Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 134 additions & 0 deletions benchmarks/benchmarks/scalability/results/expA_gpu.jsonl

Large diffs are not rendered by default.

75 changes: 75 additions & 0 deletions benchmarks/exp_A_gpu.jsonl

Large diffs are not rendered by default.

36 changes: 22 additions & 14 deletions benchmarks/resources/plots.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ def _plot_resource_file(res_file: Path, out_dir: Path,
fws = [fw for fw in FRAMEWORK_ORDER
if any(fw in cells[w] for w in workloads)]
paths: list[Path] = []
workload_device = "cuda" if suffix else "cpu"
workload_device = ("cuda" if suffix.endswith("_gpu") else "cpu")

def _vals(wl: str, fw: str, getter):
return [v for r in cells[wl].get(fw, [])
Expand Down Expand Up @@ -302,8 +302,9 @@ def _grouped(ax, getter, ylabel: str, title: str, *, scale: float = 1.0,
# 4) GPU telemetry (only when available in THIS side's rows)
gpu_rows = [r for r in rows
if r["extra"]["resources"].get("gpu", {}).get("available")]
telemetry_name = (f"resources_gpu{suffix}.png" if not suffix
else f"resources_telemetry{suffix}.png")
telemetry_name = (f"resources_telemetry{suffix}.png"
if suffix.endswith("_gpu")
else f"resources_gpu{suffix}.png")
if gpu_rows:
fig, axes = plt.subplots(1, 2, figsize=(11, 4))
g = lambda r, k: r["extra"]["resources"]["gpu"].get(k) # noqa: E731
Expand Down Expand Up @@ -368,21 +369,25 @@ def _grouped(ax, getter, ylabel: str, title: str, *, scale: float = 1.0,


def plot_resources(results_dir: Path, out_dir: Path,
suffix: str = "") -> list[Path]:
suffix: str = "", *, slug: str = "") -> list[Path]:
"""Resource-consumption figures (experiment R), both device sides.

Renders ``resources{suffix}.jsonl`` (CPU side with the default
``suffix=""``; existing figure names unchanged), then — only when no
explicit suffix is given — auto-detects ``resources_gpu.jsonl`` and
renders the GPU side with ``suffix="_gpu"``. Statistics/telemetry
availability are always evaluated within one side, never combined.
Renders ``resources{slug}.jsonl`` (CPU side; default ``slug=""`` keeps
existing figure names byte-identical), then — only when no explicit
suffix is given — auto-detects ``resources{slug}_gpu.jsonl`` and
renders the GPU side with ``{slug}_gpu`` figure suffix. Statistics/
telemetry availability are always evaluated within one side, never
combined.
"""
paths = _plot_resource_file(results_dir / f"resources{suffix}.jsonl",
out_dir, suffix)
sfx = f"_{slug}" if slug else ""
cpu_sfx = f"{sfx}{suffix}"
paths = _plot_resource_file(results_dir / f"resources{cpu_sfx}.jsonl",
out_dir, cpu_sfx)
if not suffix:
gpu_file = results_dir / "resources_gpu.jsonl"
gpu_sfx = f"{sfx}_gpu"
gpu_file = results_dir / f"resources{gpu_sfx}.jsonl"
if gpu_file.exists():
paths += _plot_resource_file(gpu_file, out_dir, "_gpu")
paths += _plot_resource_file(gpu_file, out_dir, gpu_sfx)
return paths


Expand All @@ -392,14 +397,17 @@ def build_parser() -> argparse.ArgumentParser:
default=str(REPO_ROOT / "benchmarks/resources/results"))
p.add_argument("--out", default=None,
help="figure output dir (default: <results-dir>/figures)")
p.add_argument("--env-slug", default="",
help="env slug used in result/figure filenames "
"(default '': reads resources.jsonl, unchanged)")
return p


def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
results_dir = Path(args.results_dir)
out_dir = Path(args.out) if args.out else results_dir / "figures"
paths = plot_resources(results_dir, out_dir)
paths = plot_resources(results_dir, out_dir, slug=args.env_slug)
for p in paths:
print(f"[fig] {p}")
print(f"generated {len(paths)} figures")
Expand Down
35 changes: 22 additions & 13 deletions benchmarks/resources/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,17 +146,19 @@ def _resources_table(ok: list[dict]) -> str:
return _table(header, body)


def _section_resources(results_dir: Path) -> list[str]:
def _section_resources(results_dir: Path, slug: str = "") -> list[str]:
"""Table + telemetry notes for the resource-consumption campaign (R)."""
from benchmarks.scalability.results_schema import load_rows

res_file = results_dir / "resources.jsonl"
sfx = f"_{slug}" if slug else ""
tag = f" [{slug}]" if slug else ""
res_file = results_dir / f"resources{sfx}.jsonl"
if not res_file.exists():
return []
rows_all = load_rows(res_file)
ok = [r for r in rows_all if r["status"] == "ok"]
out = [
"## Resource consumption (experiment R)", "",
"## Resource consumption (experiment R)" + tag, "",
"Fixed workloads, device CPU. CPU% = sum over the live process "
"tree (multiprocess sums may exceed 100%); RSS summed across "
"tree processes (shared pages may repeat — PSS/USS recorded in "
Expand All @@ -176,24 +178,26 @@ def _section_resources(results_dir: Path) -> list[str]:
out += [
f"- GPU telemetry: {'available (device-level only)' if gpu.get('available') else 'unavailable — ' + str(gpu.get('reason', ''))}.",
f"- Energy: {'available — ' + str(energy.get('scope', '')) if energy.get('available') else 'unavailable — ' + str(energy.get('reason', ''))}.",
"- Raw per-run rows: `results/resources.jsonl` (validation runs "
f"- Raw per-run rows: `results/resources{sfx}.jsonl` (validation runs "
"kept separately in `resources_validation.*`).", "",
]
out += _memory_subsections(ok)
return out


def _section_resources_gpu(results_dir: Path) -> list[str]:
def _section_resources_gpu(results_dir: Path, slug: str = "") -> list[str]:
"""Second side: optional GPU campaign rows (never mixed with CPU stats)."""
from benchmarks.scalability.results_schema import load_rows

res_file = results_dir / "resources_gpu.jsonl"
sfx = f"_{slug}" if slug else ""
tag = f" [{slug}]" if slug else ""
res_file = results_dir / f"resources{sfx}_gpu.jsonl"
if not res_file.exists():
return []
rows_all = load_rows(res_file)
ok = [r for r in rows_all if r["status"] == "ok"]
out = [
"## Resource consumption — GPU (device=cuda)", "",
"## Resource consumption — GPU (device=cuda)" + tag, "",
"Fixed workloads, device GPU (row `device=cuda`). Every statistic "
"below is computed from GPU rows only and never combined with the "
"CPU side above. CPU% = sum over the live process tree; RSS summed "
Expand All @@ -213,17 +217,19 @@ def _section_resources_gpu(results_dir: Path) -> list[str]:
out += [
f"- GPU telemetry (device side cuda): {'available (device-level)' if gpu.get('available') else 'unavailable — ' + str(gpu.get('reason', ''))}.",
f"- Energy (device side cuda): {'available — ' + str(energy.get('scope', '')) if energy.get('available') else 'unavailable — ' + str(energy.get('reason', ''))}.",
"- Raw per-run rows: `results/resources_gpu.jsonl`.", "",
f"- Raw per-run rows: `results/resources{sfx}_gpu.jsonl`.", "",
]
out += _memory_subsections(ok)
return out


def generate_report(results_dir: Path, out_path: Path) -> Path:
body = _section_resources(results_dir)
gpu_body = _section_resources_gpu(results_dir)
def generate_report(results_dir: Path, out_path: Path,
slug: str = "") -> Path:
body = _section_resources(results_dir, slug)
gpu_body = _section_resources_gpu(results_dir, slug)
if not body and not gpu_body:
body = ["## Resource consumption (experiment R)", "",
body = ["## Resource consumption (experiment R)"
+ (f" [{slug}]" if slug else ""), "",
"_no ok rows yet._", ""]
lines = [
"# Resource-consumption benchmark report",
Expand All @@ -246,14 +252,17 @@ def build_parser() -> argparse.ArgumentParser:
default=str(REPO_ROOT / "benchmarks/resources/results"))
p.add_argument("--out", default=None,
help="output markdown (default: <results-dir>/REPORT.md)")
p.add_argument("--env-slug", default="",
help="env slug used in result filenames "
"(default '': reads resources.jsonl, unchanged)")
return p


def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
results_dir = Path(args.results_dir)
out = Path(args.out) if args.out else results_dir / "REPORT.md"
generate_report(results_dir, out)
generate_report(results_dir, out, slug=args.env_slug)
return 0


Expand Down
40 changes: 30 additions & 10 deletions benchmarks/resources/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,16 @@

import psutil

from benchmarks.scalability.results_schema import append_row, make_row
from benchmarks.scalability.results_schema import ( # noqa: E402
append_row,
env_slug,
make_row,
)

REPO_ROOT = Path(__file__).resolve().parents[2]
FRAMEWORKS_R = ("zeroRL", "SB3", "TorchRL")
WORKLOADS = ("collect", "ppo")
DEFAULT_ENV = "LunarLander-v3"
DEFAULT_INTERVAL_S = 0.2
COLLECT_NUM_ENVS = 8
PPO_EXPERIMENT = "D"
Expand Down Expand Up @@ -646,9 +651,12 @@ def run_resource_workload(
if write_samples and mon.samples:
sdir = out.parent / "samples"
sdir.mkdir(parents=True, exist_ok=True)
# `_cuda` suffix keeps GPU samples from overwriting CPU sample files
# `_cuda` suffix keeps GPU samples from overwriting CPU sample files;
# a non-default env gets its slug prefix so campaigns can't collide.
slug_pfx = (f"{env_slug(cfg.env_id)}_"
if cfg.env_id != DEFAULT_ENV else "")
sfile = sdir / (
f"{workload}_{framework}_rep{rep}"
f"{slug_pfx}{workload}_{framework}_rep{rep}"
f"{'_cuda' if device == 'cuda' else ''}.jsonl")
sfile.write_text(
"\n".join(json.dumps(s) for s in mon.samples) + "\n",
Expand Down Expand Up @@ -727,16 +735,16 @@ def run_resource_workload(
return row


def _collect_cfg() -> Any:
def _collect_cfg(env_id: str = DEFAULT_ENV) -> Any:
from benchmarks.scalability.collection import CollectionConfig

return CollectionConfig()
return CollectionConfig(env_id=env_id)


def _ppo_cfg() -> Any:
def _ppo_cfg(env_id: str = DEFAULT_ENV) -> Any:
from benchmarks.scalability.ppo_protocols import PPOProtocolConfig

return PPOProtocolConfig()
return PPOProtocolConfig(env_id=env_id)


# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -807,9 +815,13 @@ def build_parser() -> argparse.ArgumentParser:
help="repetitions for the collect workload (A schedule)")
p.add_argument("--reps-ppo", type=int, default=3,
help="repetitions for the ppo workload (C/D schedule)")
p.add_argument("--env", default=DEFAULT_ENV,
help=f"gymnasium env id (default {DEFAULT_ENV}; a "
"non-default env defaults to resources_<slug>.jsonl)")
p.add_argument("--out", default=None,
help="JSONL output (default: "
"benchmarks/resources/results/resources.jsonl)")
"benchmarks/resources/results/resources.jsonl, "
"with _<env-slug> appended for non-default --env)")
p.add_argument("--device", choices=["cpu", "cuda"], default="cpu",
help="compute device for the workload (default cpu; "
"env stepping is always CPU)")
Expand All @@ -822,7 +834,12 @@ def build_parser() -> argparse.ArgumentParser:

def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
out = Path(args.out) if args.out else DEFAULT_OUT
if args.out:
out = Path(args.out)
elif args.env != DEFAULT_ENV:
out = DEFAULT_OUT.with_name(f"resources_{env_slug(args.env)}.jsonl")
else:
out = DEFAULT_OUT
if args.validate:
summary = run_validation()
print(json.dumps(summary, indent=2))
Expand All @@ -843,11 +860,14 @@ def main(argv: list[str] | None = None) -> int:
reps = args.reps_collect if workload == "collect" else args.reps_ppo
for fw in frameworks:
for rep in range(reps):
label = f"{fw:>7} {workload:>7} rep={rep} seed={1000 + rep}"
label = (f"{fw:>7} {workload:>7} env={args.env} "
f"rep={rep} seed={1000 + rep}")
try:
row = run_resource_workload(
fw, workload, rep, interval=args.interval,
out_path=out,
collection_cfg=_collect_cfg(args.env),
ppo_cfg=_ppo_cfg(args.env),
write_samples=not args.no_samples,
device=args.device)
except Exception:
Expand Down
Loading
Loading