From 699d28d08a72f6d3a6dcbd8df5988350118b3d76 Mon Sep 17 00:00:00 2001 From: Dar-rius Date: Wed, 4 Nov 2026 19:13:14 +0000 Subject: [PATCH 1/3] refact(factory): add encoder in factory model --- examples/pendulum.py | 2 +- zerorl/helpers/encoder.py | 37 +++++++++---------------------------- zerorl/helpers/factory.py | 21 ++++++++++++++++----- 3 files changed, 26 insertions(+), 34 deletions(-) diff --git a/examples/pendulum.py b/examples/pendulum.py index c85824c..473d4de 100644 --- a/examples/pendulum.py +++ b/examples/pendulum.py @@ -11,6 +11,6 @@ config.device = torch.device("cpu") algo_config = AlgoConfig(ent_coef=0.0) -trainer = easy_train_ppo("Pendulum-v1", config, algo_config, evals=10) +trainer = easy_train_ppo("Pendulum-v1", config, algo_config) trainer.train(use_wandb=True) trainer.try_agent() diff --git a/zerorl/helpers/encoder.py b/zerorl/helpers/encoder.py index 81aeb45..714113e 100644 --- a/zerorl/helpers/encoder.py +++ b/zerorl/helpers/encoder.py @@ -20,7 +20,6 @@ class ObservationEncoder(nn.Module): def __init__( self, obs_shape: tuple[int, ...] | int, - hidden_dim: int = 64, *, image_layout: Literal["auto", "CHW", "HWC"] = "auto", image_scale: float | None = 255.0): @@ -34,7 +33,6 @@ def __init__( self.is_image = len(self.obs_shape) == 3 self.image_scale = image_scale self.image_layout: str | None = None - self.output_dim = hidden_dim if self.is_image: if image_layout == "auto": @@ -73,30 +71,13 @@ def __init__( else: self.features = nn.Flatten(start_dim=1) #type: ignore self.n_features = math.prod(self.obs_shape) + self.output_dim = self.n_features - def get_features(self): return (self.features, self.n_features) - - - -def make_encoder( - obs_shape: tuple[int, ...] | int, - hidden_dim: int, - encoder: nn.Module | None, - encoder_output_dim: int | None, - image_layout: Literal["auto", "CHW", "HWC"], - image_scale: float | None, -) -> tuple[nn.Module, int]: - if encoder is None: - obs_encoder = ObservationEncoder( - obs_shape, - hidden_dim, - image_layout=image_layout, - image_scale=image_scale, - ) - return obs_encoder.get_features() - dim = encoder_output_dim - if dim is None: - dim = getattr(encoder, "output_dim", None) - if not isinstance(dim, int) or dim <= 0: - raise ValueError("Custom encoder requires encoder_output_dim > 0") - return encoder, dim + def forward(self, state: torch.Tensor): + x = state.float() + if self.is_image: + if self.image_layout == "HWC": + x = x.permute(0, 3, 1, 2).contiguous() + if self.image_scale is not None: + x = x / self.image_scale + return self.features(x) diff --git a/zerorl/helpers/factory.py b/zerorl/helpers/factory.py index a04f7c5..2f97e37 100644 --- a/zerorl/helpers/factory.py +++ b/zerorl/helpers/factory.py @@ -6,12 +6,13 @@ import numpy as np import torch from typing import Callable +from torch import nn from torch import Tensor from zerorl.functions import vectorize_env from zerorl.helpers.agent import BaseAgent, eval_action from zerorl.helpers.env import BaseEnv from zerorl.buffer import Buffer -from torch import nn +from zerorl.helpers.encoder import ObservationEncoder def get_env(env_id: str | Callable | BaseEnv, num_envs: int = 1, render_mode: str | None= None): @@ -125,9 +126,13 @@ def __init__(self, input_dim: int, output_dim: int, is_discrete: bool, hidden_di self.input_dim = input_dim self.output_dim = output_dim + # CNN Encoder + self.encoder = ObservationEncoder(self.input_dim) + self.encoder_dim = self.encoder.output_dim + # Feature Extractor self.extract_layer = nn.Sequential( - nn.Linear(self.input_dim, self.hidden_dim), + nn.Linear(self.encoder_dim, self.hidden_dim), nn.Tanh(), nn.Linear(self.hidden_dim, self.hidden_dim), nn.Tanh()) @@ -159,7 +164,8 @@ def _orthogonal_init(self): def forward(self, state: Tensor): """Forward pass returning (logits, value).""" - x = self.extract_layer(state) + x = self.encoder(state) + x = self.extract_layer(x) logits = self.actor(x) value = self.critic(x) return (logits, value) @@ -200,10 +206,14 @@ def __init__(self, input_dim: int, output_dim: int, is_discrete: bool, hidden_di self.hidden_dim = hidden_dim self.input_dim = input_dim self.output_dim = output_dim + + # CNN Encoder + self.encoder = ObservationEncoder(self.input_dim) + self.encoder_dim = self.encoder.output_dim # Feature Extractor self.extract_layer = nn.Sequential( - nn.Linear(self.input_dim, self.hidden_dim), + nn.Linear(self.encoder_dim, self.hidden_dim), nn.Tanh(), nn.Linear(self.hidden_dim, self.hidden_dim), nn.Tanh() @@ -232,7 +242,8 @@ def _orthogonal_init(self): def forward(self, state: Tensor): """Forward pass returning logits.""" - x = self.extract_layer(state) + x = self.encoder(state) + x = self.extract_layer(x) logits = self.actor(x) return logits From 17286bf5a31c1878acf6ae801130fb9c05b26d9c Mon Sep 17 00:00:00 2001 From: Dar-rius Date: Fri, 6 Nov 2026 13:36:55 +0000 Subject: [PATCH 2/3] refact(factory): improve encoder integration Factory DNN model --- examples/pendulum.py | 3 ++- zerorl/functions.py | 8 +++----- zerorl/helpers/encoder.py | 6 ++++-- zerorl/helpers/factory.py | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/examples/pendulum.py b/examples/pendulum.py index 473d4de..6069ae1 100644 --- a/examples/pendulum.py +++ b/examples/pendulum.py @@ -6,7 +6,8 @@ model_name="Pendulum", project_name="Pendulum-example", timestamp = 1_000_000, - num_envs = 4 + num_envs = 4, + profile = True ) config.device = torch.device("cpu") algo_config = AlgoConfig(ent_coef=0.0) diff --git a/zerorl/functions.py b/zerorl/functions.py index c3e2cc8..5c3c67b 100644 --- a/zerorl/functions.py +++ b/zerorl/functions.py @@ -1,6 +1,6 @@ """Utility functions for RL training. -Provides vectorize_env() for environment wrapping, env_step() for +Provides vectorize_env() for environment wrapping, env_step() forfunc agent-env interaction, and helpers for state processing, seeding, and action conversion. """ @@ -312,10 +312,8 @@ def get_obs_act(env: SyncVectorEnv) -> Any: else: act_n = int(np.prod(act_dim.shape)) #type: ignore - if len(obs_dim.shape) == 3: #type: ignore - obs_n = obs_dim - else: - obs_n = obs_dim.shape[-1] #type: ignore + obs_n = obs_dim.shape + return (obs_dim.shape, act_dim.shape, obs_n, act_n, is_discrete) def actions_limits(env: SyncVectorEnv, agent: BaseAgent) -> tuple[Any, Any] | None: diff --git a/zerorl/helpers/encoder.py b/zerorl/helpers/encoder.py index 714113e..22be54b 100644 --- a/zerorl/helpers/encoder.py +++ b/zerorl/helpers/encoder.py @@ -24,7 +24,7 @@ def __init__( image_layout: Literal["auto", "CHW", "HWC"] = "auto", image_scale: float | None = 255.0): super().__init__() - self.obs_shape = (obs_shape,) if isinstance(obs_shape, int) else tuple(obs_shape) + self.obs_shape = (obs_shape,) if isinstance(obs_shape, int) else obs_shape if not self.obs_shape or any(dim <= 0 for dim in self.obs_shape): raise ValueError(f"Invalid observation shape: {self.obs_shape}") if image_scale is not None and image_scale <= 0: @@ -74,7 +74,9 @@ def __init__( self.output_dim = self.n_features def forward(self, state: torch.Tensor): - x = state.float() + x = state + if x.ndim == len(self.obs_shape): + x = x.unsqueeze(0) if self.is_image: if self.image_layout == "HWC": x = x.permute(0, 3, 1, 2).contiguous() diff --git a/zerorl/helpers/factory.py b/zerorl/helpers/factory.py index 2f97e37..8b6fb23 100644 --- a/zerorl/helpers/factory.py +++ b/zerorl/helpers/factory.py @@ -118,7 +118,7 @@ class ActorCriticAgent(BaseAgent): is_discrete: Whether the action space is discrete. hidden_dim: Hidden layer size (default 64). """ - def __init__(self, input_dim: int, output_dim: int, is_discrete: bool, hidden_dim: int = 64): + def __init__(self, input_dim: tuple[int, ...] | int, output_dim: int, is_discrete: bool, hidden_dim: int = 64): super().__init__() self.is_discrete = is_discrete From 2983ae5284563ff60b991eaa4db7495f674ea33a Mon Sep 17 00:00:00 2001 From: Dar-rius Date: Fri, 6 Nov 2026 14:40:17 +0000 Subject: [PATCH 3/3] refact(factory): init encoder in orthogonal init --- benchmarks/scalability/ppo_protocols.py | 44 +++++++++++++++++++++++++ zerorl/helpers/factory.py | 13 +++++--- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/benchmarks/scalability/ppo_protocols.py b/benchmarks/scalability/ppo_protocols.py index b0d05ec..d9b1db3 100644 --- a/benchmarks/scalability/ppo_protocols.py +++ b/benchmarks/scalability/ppo_protocols.py @@ -357,6 +357,50 @@ def _dedup_params(*modules: torch.nn.Module) -> list[torch.nn.Parameter]: return out +<<<<<<< HEAD +======= +class SummedNormal(torch.distributions.Normal): + """Normal with log_prob/entropy summed over the action dim. + + Exactly zeroRL's ``eval_action`` reduction (``sum(dim=-1)`` for 2-D + outputs). torchrl's plain ``Normal`` leaves per-action-dim (B, A) + values for Tensor actions (only CompositeDistribution aggregates), + which would mis-shape ClipPPOLoss vs the working (B,) Categorical + path — this subclass keeps both paths shape-identical. + """ + + def log_prob(self, action: torch.Tensor) -> torch.Tensor: + return super().log_prob(action).sum(dim=-1) + + def entropy(self) -> torch.Tensor: + return super().entropy().sum(dim=-1) + + @property + def deterministic_sample(self) -> torch.Tensor: + """Gaussian mode == mean: torchrl's deterministic-action API. + + Provides the attribute SafeProbabilisticModule looks up so it takes + its fast path instead of warning about a mean fallback — the value + is identical either way (for a Normal, the mode is the mean). + """ + return self.mean + + +class _MeanScaleHead(torch.nn.Module): + """Continuous PPO actor params: mean from the shared actor head, scale + from the trunk's TRAINABLE log_std with zeroRL's clamp semantics.""" + + def __init__(self, trunk: TorchRLSharedTrunk) -> None: + super().__init__() + self.trunk = trunk + + def forward(self, features: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + mean = self.trunk.actor_head(features) + scale = self.trunk.log_std.clamp(-3.0, 1.0).exp().expand_as(mean) + return mean, scale + + +>>>>>>> b26e8ba (refact(factory): init encoder in orthogonal init) def build_torchrl_ppo_wiring(trunk: TorchRLSharedTrunk, cfg: PPOProtocolConfig, updates: int) -> dict[str, Any]: diff --git a/zerorl/helpers/factory.py b/zerorl/helpers/factory.py index 8b6fb23..c960af0 100644 --- a/zerorl/helpers/factory.py +++ b/zerorl/helpers/factory.py @@ -147,14 +147,19 @@ def __init__(self, input_dim: tuple[int, ...] | int, output_dim: int, is_discret self._orthogonal_init() - def _init_layer(self, layer: nn.Linear, gain: float): + def _init_layer(self, layer: nn.Linear | nn.Conv2d, gain: float): nn.init.orthogonal_(layer.weight, gain=gain) if layer.bias is not None: nn.init.constant_(layer.bias, 0.0) def _orthogonal_init(self): """Apply orthogonal weight initialization with gain based on layer role.""" - # Hidden layers - for layer in self.extract_layer: + # CNN layers + for layer in self.encoder.features.modules(): + if isinstance(layer, nn.Conv2d): + self._init_layer(layer, gain=np.sqrt(2)) + + # Extract layers + for layer in self.extract_layer.modules(): if isinstance(layer, nn.Linear): self._init_layer(layer, gain=np.sqrt(2)) # Actor output @@ -233,7 +238,7 @@ def _init_layer(self, layer: nn.Linear, gain: float): def _orthogonal_init(self): """Apply orthogonal weight initialization with gain based on layer role.""" - # Hidden layers + # Extractor layers for layer in self.extract_layer: if isinstance(layer, nn.Linear): self._init_layer(layer, gain=np.sqrt(2))