Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 0 additions & 134 deletions benchmarks/benchmarks/scalability/results/expA_gpu.jsonl

This file was deleted.

75 changes: 0 additions & 75 deletions benchmarks/exp_A_gpu.jsonl

This file was deleted.

23 changes: 17 additions & 6 deletions benchmarks/scalability/collection.py
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,10 @@ def _absorb_actions(fb: dict[str, Any], actions: list[int],
if len(fb["actions_head"]) < FINGERPRINT_HEAD:
fb["actions_head"].append(int(a) if is_discrete
else round(float(a), 6))
total = int(sum(actions))
# int actions sum to int (unchanged); continuous sums MUST stay float —
# int() truncation made every Box action sum to 0 and broke parity with
# zeroRL's float(np.sum(buffer_actions)) fingerprint.
total = sum(actions)
if tracker.target == fb.get("_warm_target"):
fb["action_sum_warmup"] += total
else:
Expand Down Expand Up @@ -381,14 +384,22 @@ def run_zerorl_collection(cfg: CollectionConfig, num_envs: int, seed: int,

# fingerprints from the (official-path) buffer
data = buffer.get_all()
actions = data["action"].reshape(-1).tolist()
flat = data["action"].reshape(-1)
# action elements per vectorized step: 1 for scalar (discrete)
# actions, act_dim for Box actions — the warmup slice must cover
# whole transitions, matching what _absorb_actions accumulates for
# SB3/TorchRL (all warmup action values), not a flat-element prefix.
act_dim = flat.numel() // (data["action"].shape[0]
* data["action"].shape[1])
actions = flat.tolist()
warm_n = cfg.warmup
warm_elems = warm_n * act_dim
fb = {
"first_obs_head": [round(float(x), 6) for x in first_obs.reshape(-1)[:4]],
"actions_head": [int(a) if is_discrete else round(float(a), 6)
for a in actions[:FINGERPRINT_HEAD]],
"action_sum_warmup": float(np.sum(actions[:warm_n])),
"action_sum_measure": float(np.sum(actions[warm_n:])),
"action_sum_warmup": float(np.sum(actions[:warm_elems])),
"action_sum_measure": float(np.sum(actions[warm_elems:])),
}

warm_t = BudgetTracker(cfg.warmup)
Expand Down Expand Up @@ -616,7 +627,7 @@ def run_torchrl_collection(cfg: CollectionConfig, num_envs: int, seed: int,
for x in td["observation"][0].reshape(-1)[:4]]
# td batch dims are (env, time); fingerprint order must be
# time-major (step0_env0, step0_env1, ...) to match zeroRL/SB3.
_absorb_actions(fb, td["action"].movedim(-1, 0).reshape(-1).tolist(),
_absorb_actions(fb, td["action"].movedim(1, 0).reshape(-1).tolist(),
warm_tracker, is_discrete=d.is_discrete)
warm_tracker.record(n)
warm_s = time.perf_counter() - t
Expand All @@ -640,7 +651,7 @@ def run_torchrl_collection(cfg: CollectionConfig, num_envs: int, seed: int,
t = time.perf_counter()
for td in collector2:
n = int(td.batch_size.numel())
_absorb_actions(fb, td["action"].movedim(-1, 0).reshape(-1).tolist(),
_absorb_actions(fb, td["action"].movedim(1, 0).reshape(-1).tolist(),
meas_tracker, is_discrete=d.is_discrete)
for key in meas_fields:
meas_fields[key] += n
Expand Down
4 changes: 2 additions & 2 deletions benchmarks/scalability/workers.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ def run_torchrl_mp_collection(cfg: CollectionConfig, num_envs: int,
if first_obs is None:
first_obs = [round(float(x), 6)
for x in td["observation"][0, 0].reshape(-1)[:4]]
_absorb_actions(fb, td["action"].movedim(-1, 0).reshape(-1).tolist(),
_absorb_actions(fb, td["action"].movedim(1, 0).reshape(-1).tolist(),
warm_tracker, is_discrete=d.is_discrete)
warm_tracker.record(n)
warm_s = time.perf_counter() - t
Expand All @@ -180,7 +180,7 @@ def run_torchrl_mp_collection(cfg: CollectionConfig, num_envs: int,
t = time.perf_counter()
for td in collector2:
n = int(td.batch_size.numel())
_absorb_actions(fb, td["action"].movedim(-1, 0).reshape(-1).tolist(),
_absorb_actions(fb, td["action"].movedim(1, 0).reshape(-1).tolist(),
meas_tracker, is_discrete=d.is_discrete)
meas_tracker.record(n)
meas_s = time.perf_counter() - t
Expand Down
44 changes: 22 additions & 22 deletions tests/test_benchmark_scaling.py
Original file line number Diff line number Diff line change
Expand Up @@ -1199,17 +1199,17 @@ def test_torchrl_ppo_e2e_trunk_count(self, monkeypatch) -> None:
policy_calls = [0]
orig_wiring = pp.build_torchrl_ppo_wiring

def wiring_spy(trunk, cfg, updates):
wiring = orig_wiring(trunk, cfg, updates)
def wiring_spy(trunk, cfg, updates, **kwargs):
wiring = orig_wiring(trunk, cfg, updates, **kwargs)
wiring["collector_policy"].register_forward_hook(
lambda *_: policy_calls.__setitem__(0, policy_calls[0] + 1))
return wiring

monkeypatch.setattr(pp, "build_torchrl_ppo_wiring", wiring_spy)

class CountingTrunk(ref.TorchRLSharedTrunk):
def __init__(self, sd=None):
super().__init__(sd)
def __init__(self, sd=None, **kwargs):
super().__init__(sd, **kwargs)
self.trunk.register_forward_hook(
lambda *_: counter.__setitem__(0, counter[0] + 1))

Expand Down Expand Up @@ -1698,28 +1698,28 @@ def boom(*_args, **_kwargs):

class TestEnvAgnosticAdaptation:
"""Env-agnostic + continuous-action adaptation of the shared reference
policy (HumanoidStandup-v4 as the continuous case)."""
policy (HumanoidStandup-v5 as the continuous case)."""

def test_env_slug_and_dims(self) -> None:
assert schema.env_slug("HumanoidStandup-v4") == "humanoidstandupv4"
assert schema.env_slug("HumanoidStandup-v5") == "humanoidstandupv5"
assert schema.env_slug("LunarLander-v3") == "lunarlanderv3"

d = ref.env_dims("LunarLander-v3")
assert (d.obs_n, d.act_n, d.is_discrete) == (8, 4, True)

d = ref.env_dims("HumanoidStandup-v4") # gym.make; lru-cached
assert d.obs_shape == (376,)
d = ref.env_dims("HumanoidStandup-v5") # gym.make; lru-cached
assert d.obs_shape == (348,)
assert d.act_n == 17
assert d.is_discrete is False

def test_humanoid_reference_sd_shapes(self) -> None:
sd = ref.build_reference_state_dict(4242, obs_dim=376, act_dim=17,
sd = ref.build_reference_state_dict(4242, obs_dim=348, act_dim=17,
is_discrete=False)
assert sd["extract_layer.0.weight"].shape == (64, 376)
assert sd["extract_layer.0.weight"].shape == (64, 348)
assert sd["actor.weight"].shape == (17, 64)
assert "log_std" in sd and sd["log_std"].shape == (17,)

trunk = ref.TorchRLSharedTrunk(sd, obs_dim=376, act_dim=17)
trunk = ref.TorchRLSharedTrunk(sd, obs_dim=348, act_dim=17)
assert isinstance(trunk.log_std, torch.nn.Parameter)
assert any(p is trunk.log_std for p in trunk.parameters())

Expand All @@ -1735,19 +1735,19 @@ def test_continuous_cross_framework_equivalence(self) -> None:

from benchmarks.scalability.collection import _TorchRLCollectPolicy

sd = ref.build_reference_state_dict(4242, obs_dim=376, act_dim=17,
sd = ref.build_reference_state_dict(4242, obs_dim=348, act_dim=17,
is_discrete=False)
agent = ref.build_deterministic_zerorl_agent(
sd, obs_dim=376, act_dim=17, is_discrete=False)
sd, obs_dim=348, act_dim=17, is_discrete=False)
sb3 = ref.build_sb3_shared_policy(
gym.spaces.Box(-1.0, 1.0, shape=(376,), dtype="float32"),
gym.spaces.Box(-1.0, 1.0, shape=(348,), dtype="float32"),
gym.spaces.Box(-1.0, 1.0, shape=(17,), dtype="float32"))
ref.load_into_sb3(sb3, sd)
trunk = ref.TorchRLSharedTrunk(sd, obs_dim=376, act_dim=17)
trunk = ref.TorchRLSharedTrunk(sd, obs_dim=348, act_dim=17)
torchrl_policy = _TorchRLCollectPolicy(trunk)

torch.manual_seed(0)
obs = torch.randn(8, 376)
obs = torch.randn(8, 348)

z = agent.get_action(obs)
s_actions, s_values, s_lp, s_ent = sb3.forward_collection(obs)
Expand Down Expand Up @@ -1777,8 +1777,8 @@ def test_env_flag_and_auto_out(self, monkeypatch, tmp_path) -> None:
for mod in (run_mod, res_run):
assert mod.build_parser().parse_args([]).env == "LunarLander-v3"
parsed = mod.build_parser().parse_args(
["--env", "HumanoidStandup-v4"])
assert parsed.env == "HumanoidStandup-v4"
["--env", "HumanoidStandup-v5"])
assert parsed.env == "HumanoidStandup-v5"

# resources report/plots expose build_parser: --env-slug default ""
# + accepted override
Expand Down Expand Up @@ -1812,7 +1812,7 @@ def stub_fn(*_args, __cap=captured, **kwargs):
assert captured == ["", "x"], (fn_name, captured)

# auto-out composition used by run.py mains for a non-default env
assert ("expA_" + schema.env_slug("HumanoidStandup-v4")
+ ".jsonl") == "expA_humanoidstandupv4.jsonl"
assert ("resources_" + schema.env_slug("HumanoidStandup-v4")
+ ".jsonl") == "resources_humanoidstandupv4.jsonl"
assert ("expA_" + schema.env_slug("HumanoidStandup-v5")
+ ".jsonl") == "expA_humanoidstandupv5.jsonl"
assert ("resources_" + schema.env_slug("HumanoidStandup-v5")
+ ".jsonl") == "resources_humanoidstandupv5.jsonl"
8 changes: 4 additions & 4 deletions tests/test_resources.py
Original file line number Diff line number Diff line change
Expand Up @@ -674,8 +674,8 @@ def test_resources_env_flag_and_slug() -> None:

args = build_parser().parse_args([])
assert args.env == "LunarLander-v3"
args = build_parser().parse_args(["--env", "HumanoidStandup-v4"])
assert args.env == "HumanoidStandup-v4"
args = build_parser().parse_args(["--env", "HumanoidStandup-v5"])
assert args.env == "HumanoidStandup-v5"
# non-default env => slug-prefixed sample files (run.py composition)
assert f"{env_slug('HumanoidStandup-v4')}_collect_zeroRL_rep0.jsonl" == \
"humanoidstandupv4_collect_zeroRL_rep0.jsonl"
assert f"{env_slug('HumanoidStandup-v5')}_collect_zeroRL_rep0.jsonl" == \
"humanoidstandupv5_collect_zeroRL_rep0.jsonl"
7 changes: 5 additions & 2 deletions zerorl/functions.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""Utility functions for RL training.

Provides vectorize_env() for environment wrapping, env_step() forfunc
Provides vectorize_env() for environment wrapping, env_step() for
agent-env interaction, and helpers for state processing, seeding,
and action conversion.
"""
Expand Down Expand Up @@ -312,7 +312,10 @@ def get_obs_act(env: SyncVectorEnv) -> Any:
else:
act_n = int(np.prod(act_dim.shape)) #type: ignore

obs_n = obs_dim.shape
if len(obs_dim.shape) == 3: # type: ignore
obs_n = obs_dim # image spaces: pass the Space itself
else:
obs_n = obs_dim.shape[-1] # type: ignore # flat obs dim (int)

return (obs_dim.shape, act_dim.shape, obs_n, act_n, is_discrete)

Expand Down
Loading