diff --git a/benchmarks/scalability/ppo_protocols.py b/benchmarks/scalability/ppo_protocols.py index c268550..695d3e8 100644 --- a/benchmarks/scalability/ppo_protocols.py +++ b/benchmarks/scalability/ppo_protocols.py @@ -363,7 +363,6 @@ def _dedup_params(*modules: torch.nn.Module) -> list[torch.nn.Parameter]: out.append(p) return out - class SummedNormal(torch.distributions.Normal): """Normal with log_prob/entropy summed over the action dim. @@ -380,6 +379,15 @@ def log_prob(self, action: torch.Tensor) -> torch.Tensor: def entropy(self) -> torch.Tensor: return super().entropy().sum(dim=-1) + @property + def deterministic_sample(self) -> torch.Tensor: + """Gaussian mode == mean: torchrl's deterministic-action API. + + Provides the attribute SafeProbabilisticModule looks up so it takes + its fast path instead of warning about a mean fallback — the value + is identical either way (for a Normal, the mode is the mean). + """ + return self.mean class _MeanScaleHead(torch.nn.Module): """Continuous PPO actor params: mean from the shared actor head, scale diff --git a/examples/pendulum.py b/examples/pendulum.py index c85824c..6069ae1 100644 --- a/examples/pendulum.py +++ b/examples/pendulum.py @@ -6,11 +6,12 @@ model_name="Pendulum", project_name="Pendulum-example", timestamp = 1_000_000, - num_envs = 4 + num_envs = 4, + profile = True ) config.device = torch.device("cpu") algo_config = AlgoConfig(ent_coef=0.0) -trainer = easy_train_ppo("Pendulum-v1", config, algo_config, evals=10) +trainer = easy_train_ppo("Pendulum-v1", config, algo_config) trainer.train(use_wandb=True) trainer.try_agent() diff --git a/zerorl/functions.py b/zerorl/functions.py index c3e2cc8..5c3c67b 100644 --- a/zerorl/functions.py +++ b/zerorl/functions.py @@ -1,6 +1,6 @@ """Utility functions for RL training. -Provides vectorize_env() for environment wrapping, env_step() for +Provides vectorize_env() for environment wrapping, env_step() forfunc agent-env interaction, and helpers for state processing, seeding, and action conversion. """ @@ -312,10 +312,8 @@ def get_obs_act(env: SyncVectorEnv) -> Any: else: act_n = int(np.prod(act_dim.shape)) #type: ignore - if len(obs_dim.shape) == 3: #type: ignore - obs_n = obs_dim - else: - obs_n = obs_dim.shape[-1] #type: ignore + obs_n = obs_dim.shape + return (obs_dim.shape, act_dim.shape, obs_n, act_n, is_discrete) def actions_limits(env: SyncVectorEnv, agent: BaseAgent) -> tuple[Any, Any] | None: diff --git a/zerorl/helpers/encoder.py b/zerorl/helpers/encoder.py index 81aeb45..22be54b 100644 --- a/zerorl/helpers/encoder.py +++ b/zerorl/helpers/encoder.py @@ -20,12 +20,11 @@ class ObservationEncoder(nn.Module): def __init__( self, obs_shape: tuple[int, ...] | int, - hidden_dim: int = 64, *, image_layout: Literal["auto", "CHW", "HWC"] = "auto", image_scale: float | None = 255.0): super().__init__() - self.obs_shape = (obs_shape,) if isinstance(obs_shape, int) else tuple(obs_shape) + self.obs_shape = (obs_shape,) if isinstance(obs_shape, int) else obs_shape if not self.obs_shape or any(dim <= 0 for dim in self.obs_shape): raise ValueError(f"Invalid observation shape: {self.obs_shape}") if image_scale is not None and image_scale <= 0: @@ -34,7 +33,6 @@ def __init__( self.is_image = len(self.obs_shape) == 3 self.image_scale = image_scale self.image_layout: str | None = None - self.output_dim = hidden_dim if self.is_image: if image_layout == "auto": @@ -73,30 +71,15 @@ def __init__( else: self.features = nn.Flatten(start_dim=1) #type: ignore self.n_features = math.prod(self.obs_shape) + self.output_dim = self.n_features - def get_features(self): return (self.features, self.n_features) - - - -def make_encoder( - obs_shape: tuple[int, ...] | int, - hidden_dim: int, - encoder: nn.Module | None, - encoder_output_dim: int | None, - image_layout: Literal["auto", "CHW", "HWC"], - image_scale: float | None, -) -> tuple[nn.Module, int]: - if encoder is None: - obs_encoder = ObservationEncoder( - obs_shape, - hidden_dim, - image_layout=image_layout, - image_scale=image_scale, - ) - return obs_encoder.get_features() - dim = encoder_output_dim - if dim is None: - dim = getattr(encoder, "output_dim", None) - if not isinstance(dim, int) or dim <= 0: - raise ValueError("Custom encoder requires encoder_output_dim > 0") - return encoder, dim + def forward(self, state: torch.Tensor): + x = state + if x.ndim == len(self.obs_shape): + x = x.unsqueeze(0) + if self.is_image: + if self.image_layout == "HWC": + x = x.permute(0, 3, 1, 2).contiguous() + if self.image_scale is not None: + x = x / self.image_scale + return self.features(x) diff --git a/zerorl/helpers/factory.py b/zerorl/helpers/factory.py index a04f7c5..c960af0 100644 --- a/zerorl/helpers/factory.py +++ b/zerorl/helpers/factory.py @@ -6,12 +6,13 @@ import numpy as np import torch from typing import Callable +from torch import nn from torch import Tensor from zerorl.functions import vectorize_env from zerorl.helpers.agent import BaseAgent, eval_action from zerorl.helpers.env import BaseEnv from zerorl.buffer import Buffer -from torch import nn +from zerorl.helpers.encoder import ObservationEncoder def get_env(env_id: str | Callable | BaseEnv, num_envs: int = 1, render_mode: str | None= None): @@ -117,7 +118,7 @@ class ActorCriticAgent(BaseAgent): is_discrete: Whether the action space is discrete. hidden_dim: Hidden layer size (default 64). """ - def __init__(self, input_dim: int, output_dim: int, is_discrete: bool, hidden_dim: int = 64): + def __init__(self, input_dim: tuple[int, ...] | int, output_dim: int, is_discrete: bool, hidden_dim: int = 64): super().__init__() self.is_discrete = is_discrete @@ -125,9 +126,13 @@ def __init__(self, input_dim: int, output_dim: int, is_discrete: bool, hidden_di self.input_dim = input_dim self.output_dim = output_dim + # CNN Encoder + self.encoder = ObservationEncoder(self.input_dim) + self.encoder_dim = self.encoder.output_dim + # Feature Extractor self.extract_layer = nn.Sequential( - nn.Linear(self.input_dim, self.hidden_dim), + nn.Linear(self.encoder_dim, self.hidden_dim), nn.Tanh(), nn.Linear(self.hidden_dim, self.hidden_dim), nn.Tanh()) @@ -142,14 +147,19 @@ def __init__(self, input_dim: int, output_dim: int, is_discrete: bool, hidden_di self._orthogonal_init() - def _init_layer(self, layer: nn.Linear, gain: float): + def _init_layer(self, layer: nn.Linear | nn.Conv2d, gain: float): nn.init.orthogonal_(layer.weight, gain=gain) if layer.bias is not None: nn.init.constant_(layer.bias, 0.0) def _orthogonal_init(self): """Apply orthogonal weight initialization with gain based on layer role.""" - # Hidden layers - for layer in self.extract_layer: + # CNN layers + for layer in self.encoder.features.modules(): + if isinstance(layer, nn.Conv2d): + self._init_layer(layer, gain=np.sqrt(2)) + + # Extract layers + for layer in self.extract_layer.modules(): if isinstance(layer, nn.Linear): self._init_layer(layer, gain=np.sqrt(2)) # Actor output @@ -159,7 +169,8 @@ def _orthogonal_init(self): def forward(self, state: Tensor): """Forward pass returning (logits, value).""" - x = self.extract_layer(state) + x = self.encoder(state) + x = self.extract_layer(x) logits = self.actor(x) value = self.critic(x) return (logits, value) @@ -200,10 +211,14 @@ def __init__(self, input_dim: int, output_dim: int, is_discrete: bool, hidden_di self.hidden_dim = hidden_dim self.input_dim = input_dim self.output_dim = output_dim + + # CNN Encoder + self.encoder = ObservationEncoder(self.input_dim) + self.encoder_dim = self.encoder.output_dim # Feature Extractor self.extract_layer = nn.Sequential( - nn.Linear(self.input_dim, self.hidden_dim), + nn.Linear(self.encoder_dim, self.hidden_dim), nn.Tanh(), nn.Linear(self.hidden_dim, self.hidden_dim), nn.Tanh() @@ -223,7 +238,7 @@ def _init_layer(self, layer: nn.Linear, gain: float): def _orthogonal_init(self): """Apply orthogonal weight initialization with gain based on layer role.""" - # Hidden layers + # Extractor layers for layer in self.extract_layer: if isinstance(layer, nn.Linear): self._init_layer(layer, gain=np.sqrt(2)) @@ -232,7 +247,8 @@ def _orthogonal_init(self): def forward(self, state: Tensor): """Forward pass returning logits.""" - x = self.extract_layer(state) + x = self.encoder(state) + x = self.extract_layer(x) logits = self.actor(x) return logits