Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion benchmarks/scalability/ppo_protocols.py
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,6 @@ def _dedup_params(*modules: torch.nn.Module) -> list[torch.nn.Parameter]:
out.append(p)
return out


class SummedNormal(torch.distributions.Normal):
"""Normal with log_prob/entropy summed over the action dim.

Expand All @@ -380,6 +379,15 @@ def log_prob(self, action: torch.Tensor) -> torch.Tensor:
def entropy(self) -> torch.Tensor:
return super().entropy().sum(dim=-1)

@property
def deterministic_sample(self) -> torch.Tensor:
"""Gaussian mode == mean: torchrl's deterministic-action API.

Provides the attribute SafeProbabilisticModule looks up so it takes
its fast path instead of warning about a mean fallback — the value
is identical either way (for a Normal, the mode is the mean).
"""
return self.mean

class _MeanScaleHead(torch.nn.Module):
"""Continuous PPO actor params: mean from the shared actor head, scale
Expand Down
5 changes: 3 additions & 2 deletions examples/pendulum.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,12 @@
model_name="Pendulum",
project_name="Pendulum-example",
timestamp = 1_000_000,
num_envs = 4
num_envs = 4,
profile = True
)
config.device = torch.device("cpu")
algo_config = AlgoConfig(ent_coef=0.0)

trainer = easy_train_ppo("Pendulum-v1", config, algo_config, evals=10)
trainer = easy_train_ppo("Pendulum-v1", config, algo_config)
trainer.train(use_wandb=True)
trainer.try_agent()
8 changes: 3 additions & 5 deletions zerorl/functions.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""Utility functions for RL training.

Provides vectorize_env() for environment wrapping, env_step() for
Provides vectorize_env() for environment wrapping, env_step() forfunc
agent-env interaction, and helpers for state processing, seeding,
and action conversion.
"""
Expand Down Expand Up @@ -312,10 +312,8 @@ def get_obs_act(env: SyncVectorEnv) -> Any:
else:
act_n = int(np.prod(act_dim.shape)) #type: ignore

if len(obs_dim.shape) == 3: #type: ignore
obs_n = obs_dim
else:
obs_n = obs_dim.shape[-1] #type: ignore
obs_n = obs_dim.shape

return (obs_dim.shape, act_dim.shape, obs_n, act_n, is_discrete)

def actions_limits(env: SyncVectorEnv, agent: BaseAgent) -> tuple[Any, Any] | None:
Expand Down
41 changes: 12 additions & 29 deletions zerorl/helpers/encoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,11 @@ class ObservationEncoder(nn.Module):
def __init__(
self,
obs_shape: tuple[int, ...] | int,
hidden_dim: int = 64,
*,
image_layout: Literal["auto", "CHW", "HWC"] = "auto",
image_scale: float | None = 255.0):
super().__init__()
self.obs_shape = (obs_shape,) if isinstance(obs_shape, int) else tuple(obs_shape)
self.obs_shape = (obs_shape,) if isinstance(obs_shape, int) else obs_shape
if not self.obs_shape or any(dim <= 0 for dim in self.obs_shape):
raise ValueError(f"Invalid observation shape: {self.obs_shape}")
if image_scale is not None and image_scale <= 0:
Expand All @@ -34,7 +33,6 @@ def __init__(
self.is_image = len(self.obs_shape) == 3
self.image_scale = image_scale
self.image_layout: str | None = None
self.output_dim = hidden_dim

if self.is_image:
if image_layout == "auto":
Expand Down Expand Up @@ -73,30 +71,15 @@ def __init__(
else:
self.features = nn.Flatten(start_dim=1) #type: ignore
self.n_features = math.prod(self.obs_shape)
self.output_dim = self.n_features

def get_features(self): return (self.features, self.n_features)



def make_encoder(
obs_shape: tuple[int, ...] | int,
hidden_dim: int,
encoder: nn.Module | None,
encoder_output_dim: int | None,
image_layout: Literal["auto", "CHW", "HWC"],
image_scale: float | None,
) -> tuple[nn.Module, int]:
if encoder is None:
obs_encoder = ObservationEncoder(
obs_shape,
hidden_dim,
image_layout=image_layout,
image_scale=image_scale,
)
return obs_encoder.get_features()
dim = encoder_output_dim
if dim is None:
dim = getattr(encoder, "output_dim", None)
if not isinstance(dim, int) or dim <= 0:
raise ValueError("Custom encoder requires encoder_output_dim > 0")
return encoder, dim
def forward(self, state: torch.Tensor):
x = state
if x.ndim == len(self.obs_shape):
x = x.unsqueeze(0)
if self.is_image:
if self.image_layout == "HWC":
x = x.permute(0, 3, 1, 2).contiguous()
if self.image_scale is not None:
x = x / self.image_scale
return self.features(x)
36 changes: 26 additions & 10 deletions zerorl/helpers/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,13 @@
import numpy as np
import torch
from typing import Callable
from torch import nn
from torch import Tensor
from zerorl.functions import vectorize_env
from zerorl.helpers.agent import BaseAgent, eval_action
from zerorl.helpers.env import BaseEnv
from zerorl.buffer import Buffer
from torch import nn
from zerorl.helpers.encoder import ObservationEncoder


def get_env(env_id: str | Callable | BaseEnv, num_envs: int = 1, render_mode: str | None= None):
Expand Down Expand Up @@ -117,17 +118,21 @@ class ActorCriticAgent(BaseAgent):
is_discrete: Whether the action space is discrete.
hidden_dim: Hidden layer size (default 64).
"""
def __init__(self, input_dim: int, output_dim: int, is_discrete: bool, hidden_dim: int = 64):
def __init__(self, input_dim: tuple[int, ...] | int, output_dim: int, is_discrete: bool, hidden_dim: int = 64):
super().__init__()

self.is_discrete = is_discrete
self.hidden_dim = hidden_dim
self.input_dim = input_dim
self.output_dim = output_dim

# CNN Encoder
self.encoder = ObservationEncoder(self.input_dim)
self.encoder_dim = self.encoder.output_dim

# Feature Extractor
self.extract_layer = nn.Sequential(
nn.Linear(self.input_dim, self.hidden_dim),
nn.Linear(self.encoder_dim, self.hidden_dim),
nn.Tanh(),
nn.Linear(self.hidden_dim, self.hidden_dim),
nn.Tanh())
Expand All @@ -142,14 +147,19 @@ def __init__(self, input_dim: int, output_dim: int, is_discrete: bool, hidden_di
self._orthogonal_init()


def _init_layer(self, layer: nn.Linear, gain: float):
def _init_layer(self, layer: nn.Linear | nn.Conv2d, gain: float):
nn.init.orthogonal_(layer.weight, gain=gain)
if layer.bias is not None: nn.init.constant_(layer.bias, 0.0)

def _orthogonal_init(self):
"""Apply orthogonal weight initialization with gain based on layer role."""
# Hidden layers
for layer in self.extract_layer:
# CNN layers
for layer in self.encoder.features.modules():
if isinstance(layer, nn.Conv2d):
self._init_layer(layer, gain=np.sqrt(2))

# Extract layers
for layer in self.extract_layer.modules():
if isinstance(layer, nn.Linear):
self._init_layer(layer, gain=np.sqrt(2))
# Actor output
Expand All @@ -159,7 +169,8 @@ def _orthogonal_init(self):

def forward(self, state: Tensor):
"""Forward pass returning (logits, value)."""
x = self.extract_layer(state)
x = self.encoder(state)
x = self.extract_layer(x)
logits = self.actor(x)
value = self.critic(x)
return (logits, value)
Expand Down Expand Up @@ -200,10 +211,14 @@ def __init__(self, input_dim: int, output_dim: int, is_discrete: bool, hidden_di
self.hidden_dim = hidden_dim
self.input_dim = input_dim
self.output_dim = output_dim

# CNN Encoder
self.encoder = ObservationEncoder(self.input_dim)
self.encoder_dim = self.encoder.output_dim

# Feature Extractor
self.extract_layer = nn.Sequential(
nn.Linear(self.input_dim, self.hidden_dim),
nn.Linear(self.encoder_dim, self.hidden_dim),
nn.Tanh(),
nn.Linear(self.hidden_dim, self.hidden_dim),
nn.Tanh()
Expand All @@ -223,7 +238,7 @@ def _init_layer(self, layer: nn.Linear, gain: float):

def _orthogonal_init(self):
"""Apply orthogonal weight initialization with gain based on layer role."""
# Hidden layers
# Extractor layers
for layer in self.extract_layer:
if isinstance(layer, nn.Linear):
self._init_layer(layer, gain=np.sqrt(2))
Expand All @@ -232,7 +247,8 @@ def _orthogonal_init(self):

def forward(self, state: Tensor):
"""Forward pass returning logits."""
x = self.extract_layer(state)
x = self.encoder(state)
x = self.extract_layer(x)
logits = self.actor(x)
return logits

Expand Down
Loading