From 4ff347aeaac7f0ba2adb320b235fb10ad557f096 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Thu, 6 Aug 2026 12:56:45 +0000 Subject: [PATCH 01/52] fix(reproduce): configurable per-step timeout (default none) + kill the whole tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `pipeline_executor` hard-coded `subprocess.run(..., shell=True, timeout=3600)`. Two defects: - **Not configurable** — a 1h wall-clock cap makes a row reproducible only on hardware at least as fast as the machine that made it, with no override. - **Orphans the workload** — with `shell=True`, a timeout kills only the shell, so the grandchild (e.g. `train.py`) keeps running past the declared failure — a false failure plus a silent GPU-cost leak on someone else's bill. Now: `--step-timeout ` / `ROAR_REPRODUCE_STEP_TIMEOUT`, **default none (no timeout)**. The step runs in its own session (`start_new_session=True`); on timeout the whole process group is SIGKILLed and reaped. Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/application/reproduce/requests.py | 2 + roar/application/reproduce/service.py | 4 +- roar/cli/commands/reproduce.py | 13 +++++ .../reproduction/pipeline_executor.py | 51 ++++++++++++++++--- 4 files changed, 61 insertions(+), 9 deletions(-) diff --git a/roar/application/reproduce/requests.py b/roar/application/reproduce/requests.py index 76841191..10290c21 100644 --- a/roar/application/reproduce/requests.py +++ b/roar/application/reproduce/requests.py @@ -20,6 +20,8 @@ class ReproduceRequest: package_sync: bool = False list_requirements: bool = False out_path: str | None = None + # Per-step wall-clock timeout in seconds for --run; None means no timeout. + step_timeout: int | None = None # Skip publish (`roar put`) steps — for third-party reproduction that # rebuilds the artifact without re-publishing to the owner's destination. no_puts: bool = False diff --git a/roar/application/reproduce/service.py b/roar/application/reproduce/service.py index 923b0e87..32672eec 100644 --- a/roar/application/reproduce/service.py +++ b/roar/application/reproduce/service.py @@ -163,7 +163,9 @@ def reproduce_artifact( return with _reproduction_session(environment.repo_dir, output): - steps_run, steps_total = PipelineExecutor(presenter=output).execute( + steps_run, steps_total = PipelineExecutor( + presenter=output, step_timeout=request.step_timeout + ).execute( pipeline, environment, request.auto_confirm, diff --git a/roar/cli/commands/reproduce.py b/roar/cli/commands/reproduce.py index be2da008..b5aedfb2 100644 --- a/roar/cli/commands/reproduce.py +++ b/roar/cli/commands/reproduce.py @@ -59,6 +59,17 @@ default=None, help="Dump DAG lineage response to a JSON file", ) +@click.option( + "--step-timeout", + "step_timeout", + type=int, + default=None, + envvar="ROAR_REPRODUCE_STEP_TIMEOUT", + help="Per-step wall-clock timeout in seconds for --run. Default: no timeout " + "(a step may be slower on the reproducing host than on the one that made it). " + "Also settable via ROAR_REPRODUCE_STEP_TIMEOUT. On timeout the whole process " + "group is killed so no orphaned workload keeps burning compute.", +) @click.pass_obj def reproduce( ctx: RoarContext, @@ -73,6 +84,7 @@ def reproduce( package_sync: bool, list_requirements: bool, out_path: str | None, + step_timeout: int | None, ) -> None: """Reproduce an artifact or lineage from a recorded hash. @@ -117,6 +129,7 @@ def reproduce( package_sync=package_sync, list_requirements=list_requirements, out_path=out_path, + step_timeout=step_timeout, ) ) except ValueError as exc: diff --git a/roar/execution/reproduction/pipeline_executor.py b/roar/execution/reproduction/pipeline_executor.py index 342b84f1..eba87893 100644 --- a/roar/execution/reproduction/pipeline_executor.py +++ b/roar/execution/reproduction/pipeline_executor.py @@ -8,6 +8,7 @@ import json import os import shutil +import signal import subprocess import sys from typing import TYPE_CHECKING @@ -39,6 +40,7 @@ def __init__( self, presenter: "IPresenter | None" = None, roar_executable: str | None = None, + step_timeout: int | None = None, ): """ Initialize pipeline executor. @@ -46,10 +48,13 @@ def __init__( Args: presenter: Presenter for user feedback roar_executable: Path to roar executable (auto-detected if not provided) + step_timeout: Per-step wall-clock timeout in seconds; ``None`` (default) + means no timeout. """ self._presenter = presenter or NullPresenter() self._roar_initialized = False self._roar_executable = roar_executable or self._detect_roar_executable() + self._step_timeout = step_timeout def execute( self, @@ -153,31 +158,61 @@ def _run_step( # Set up environment env = self._prepare_environment(environment, env_vars=step_env_vars) - # Run the command + # Run the command in its own session/process group so that, if a timeout + # fires, we can kill the whole tree. shell=True means the direct child is + # a shell whose grandchild (e.g. train.py) would be orphaned by a plain + # kill of the shell — leaving a workload running on the GPU past the + # declared failure. `timeout` defaults to None (no timeout): a run should + # not be capped at an arbitrary wall-clock that also makes the row only + # reproducible on hardware at least as fast as the machine that made it. try: # Note: Using shell=True for complex commands with pipes, etc. - result = subprocess.run( + proc = subprocess.Popen( wrapped_command, shell=True, cwd=environment.repo_dir, env=env, - timeout=3600, # 1 hour timeout + start_new_session=True, ) + try: + returncode = proc.wait(timeout=self._step_timeout) + except subprocess.TimeoutExpired: + self._print( + f" Step timed out after {self._step_timeout}s — " + "killing the process group" + ) + self._kill_process_group(proc) + return False - if result.returncode == 0: + if returncode == 0: self._print(" Success") return True else: - self._print(f" Failed with exit code {result.returncode}") + self._print(f" Failed with exit code {returncode}") return False - except subprocess.TimeoutExpired: - self._print(" Step timed out after 1 hour") - return False except Exception as e: self._print(f" Error: {e}") return False + @staticmethod + def _kill_process_group(proc: "subprocess.Popen[bytes]") -> None: + """SIGKILL the step's whole process group, then reap it. + + With ``shell=True`` the workload is a grandchild of the shell, so killing + only ``proc`` leaves it orphaned (a false failure + a silent GPU-cost + leak on someone else's bill). ``start_new_session=True`` gives the step + its own group, which we kill here. + """ + try: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + except (ProcessLookupError, PermissionError): + proc.kill() + try: + proc.wait(timeout=30) + except subprocess.TimeoutExpired: + pass + def _wrap_with_roar( self, command: str, From b2dac75ae177003ffc9fbdd84a6764fabb337f72 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Thu, 6 Aug 2026 13:02:45 +0000 Subject: [PATCH 02/52] fix(reproduce): fail (don't report success) when a recorded pip pin can't install MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `install_pip_packages` demoted every un-installable pin to a *warning* and returned True; `setup()` then ignored that boolean and returned a healthy EnvironmentInfo. Result: "Pip package installation complete" → "Environment ready" → a dead run (ModuleNotFoundError). The AMI re-bake only removed the torch-family instance; the class (yanked version, private package, extra-index pin) still produced a green banner + dead run. - installers: return False when a recorded pin is unresolved (fallback failed, or skipped because --pip-any-version wasn't given). - environment_setup.setup(): raise RuntimeError on a failed install instead of ignoring `success` — the reproduce service already reports RuntimeError as "Environment setup failed" (not "Environment ready"). Bypass + debug (the two points raised): - **Bypass** the pin check with the existing `--pip-any-version` (installs available versions; recorded as warnings). Default is now honest-fail. - **Debug/export** with new `--export-requirements `: writes the recorded pip pins to a pip-native requirements.txt (no uv assumption) so you can `pip install --dry-run -r ` to see exactly which pins don't resolve. Complements `--script` (which emits the shell, not the packages). Header notes the index-url/extra-index-url isn't replayed yet (that's the deeper [93]-class capture gap — a follow-up). Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/application/reproduce/requests.py | 3 +++ roar/application/reproduce/service.py | 26 +++++++++++++++++++ roar/cli/commands/reproduce.py | 11 ++++++++ .../reproduction/environment_setup.py | 11 ++++++++ roar/execution/reproduction/installers.py | 20 ++++++++++++++ 5 files changed, 71 insertions(+) diff --git a/roar/application/reproduce/requests.py b/roar/application/reproduce/requests.py index 76841191..42598857 100644 --- a/roar/application/reproduce/requests.py +++ b/roar/application/reproduce/requests.py @@ -20,6 +20,9 @@ class ReproduceRequest: package_sync: bool = False list_requirements: bool = False out_path: str | None = None + # Write the recorded pip pins to a requirements.txt (for debugging a failed + # install) instead of previewing/running; None means don't export. + export_requirements: str | None = None # Skip publish (`roar put`) steps — for third-party reproduction that # rebuilds the artifact without re-publishing to the owner's destination. no_puts: bool = False diff --git a/roar/application/reproduce/service.py b/roar/application/reproduce/service.py index 923b0e87..879eb513 100644 --- a/roar/application/reproduce/service.py +++ b/roar/application/reproduce/service.py @@ -91,6 +91,10 @@ def reproduce_artifact( hash_prefix=request.hash_prefix, ) + if request.export_requirements: + _export_pip_requirements(pipeline, request.export_requirements, output) + return + if not request.run_pipeline: _render_preview_summary( preview, @@ -483,6 +487,28 @@ def build_reproduction_script( return "\n".join(lines) +def _export_pip_requirements(pipeline: PipelineInfo, path: str, output: IPresenter) -> None: + """Write the recorded pip pins to a requirements.txt for offline debugging. + + Complements ``--script`` (which emits the reproduction shell) by emitting the + *packages* in a pip-native form the user can try directly: + ``pip install --dry-run -r `` shows exactly which pins don't resolve + (yanked, private, or on an extra index). No ``uv`` assumption. + """ + summary = PipelineMetadataParser().summarize_requirements( + pipeline.build_steps, pipeline.run_steps + ) + target = pipeline.artifact_hash or pipeline.session_hash or "" + header = [ + f"# roar reproduce — recorded pip pins for {target[:12]}", + "# Try: pip install --dry-run -r (shows which pins do not resolve)", + "# NOTE: the recorded --index-url/--extra-index-url is not replayed here yet,", + "# so a pin published only on a custom index reads as 'not found'.", + ] + Path(path).write_text("\n".join([*header, *sorted(summary.pip)]) + "\n", encoding="utf-8") + output.print(f"Wrote {len(summary.pip)} pip requirement(s) to {path}") + + def build_preview_summary( pipeline: PipelineInfo, *, diff --git a/roar/cli/commands/reproduce.py b/roar/cli/commands/reproduce.py index be2da008..515608ee 100644 --- a/roar/cli/commands/reproduce.py +++ b/roar/cli/commands/reproduce.py @@ -59,6 +59,15 @@ default=None, help="Dump DAG lineage response to a JSON file", ) +@click.option( + "--export-requirements", + "export_requirements", + type=click.Path(), + default=None, + help="Write the recorded pip pins to a requirements.txt and exit (no run). " + "Debug a failed install with `pip install --dry-run -r ` to see which " + "pins don't resolve (yanked, private, or extra-index).", +) @click.pass_obj def reproduce( ctx: RoarContext, @@ -73,6 +82,7 @@ def reproduce( package_sync: bool, list_requirements: bool, out_path: str | None, + export_requirements: str | None, ) -> None: """Reproduce an artifact or lineage from a recorded hash. @@ -117,6 +127,7 @@ def reproduce( package_sync=package_sync, list_requirements=list_requirements, out_path=out_path, + export_requirements=export_requirements, ) ) except ValueError as exc: diff --git a/roar/execution/reproduction/environment_setup.py b/roar/execution/reproduction/environment_setup.py index b47cef9d..8a0352ec 100644 --- a/roar/execution/reproduction/environment_setup.py +++ b/roar/execution/reproduction/environment_setup.py @@ -212,6 +212,17 @@ def setup_in_place( if pip_warnings: for w in pip_warnings: self.logger.warning(w) + # Propagate a failed install instead of returning a healthy-looking + # EnvironmentInfo. Previously `success` was ignored, so an unresolved + # pin still produced "Environment ready" followed by a dead run. The + # reproduce service catches RuntimeError as "Environment setup failed". + if not success: + raise RuntimeError( + "Required pip packages from the recorded provenance could not be " + "installed — the reproduction environment is incomplete. Re-run with " + "--pip-any-version to install available versions, or " + "--export-requirements to inspect/try the exact pins yourself." + ) self.logger.debug("pip installation complete") self.logger.debug("Environment setup complete") diff --git a/roar/execution/reproduction/installers.py b/roar/execution/reproduction/installers.py index 37d4e978..5a4e5c49 100644 --- a/roar/execution/reproduction/installers.py +++ b/roar/execution/reproduction/installers.py @@ -253,6 +253,7 @@ def install_packages( presenter: IPresenter | None = None, ) -> tuple[bool, list[str]]: warnings: list[str] = [] + unresolved_packages: list[str] = [] active_presenter = presenter or self._presenter if not packages: self._print("No packages to install from provenance.") @@ -302,6 +303,7 @@ def install_packages( warnings.append( f"Some pip packages failed to install: {(fallback.stderr or '').strip()}" ) + unresolved_packages = list(failed_packages) else: for package in failed_packages: warnings.append( @@ -310,6 +312,24 @@ def install_packages( else: for package in failed_packages: warnings.append(f"Skipped {package} (exact version not found)") + unresolved_packages = list(failed_packages) + + if unresolved_packages: + # A recorded pin could not be installed. Do NOT report success — that + # produced a green "Pip package installation complete" / "Environment + # ready" banner followed by a dead run (ModuleNotFoundError). Fail + # honestly so the reproduction reports the env-setup failure instead. + self._print( + f"\nEnvironment is NOT reproducible: {len(unresolved_packages)} recorded " + "pip package(s) could not be installed:" + ) + for package in unresolved_packages: + self._print(f" - {package}") + self._print( + "Re-run with --pip-any-version to install available versions instead, " + "or --export-requirements to inspect/try the exact pins yourself." + ) + return False, warnings self._print("Pip package installation complete") return True, warnings From 5d1fcc055e5025c007391e6542f815abc019c580 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Thu, 6 Aug 2026 13:16:18 +0000 Subject: [PATCH 03/52] =?UTF-8?q?test(reproduce):=20P0-1=20=E2=80=94=20hon?= =?UTF-8?q?est-fail=20on=20unresolvable=20pin=20+=20export;=20update=20dec?= =?UTF-8?q?lines-fallback=20contract?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/unit/test_environment_setup.py | 8 +- tests/unit/test_reproduce_pin_failure.py | 98 ++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_reproduce_pin_failure.py diff --git a/tests/unit/test_environment_setup.py b/tests/unit/test_environment_setup.py index 63016926..b290abc3 100644 --- a/tests/unit/test_environment_setup.py +++ b/tests/unit/test_environment_setup.py @@ -697,8 +697,10 @@ def test_prompts_user_for_fallback_when_version_unavailable(self, service, tmp_p "Install available versions instead?", default=True ) - def test_skips_failed_packages_when_user_declines_fallback(self, service, tmp_path): - """When user declines, skip the failed packages with warning.""" + def test_declining_fallback_on_missing_pin_fails(self, service, tmp_path): + """When the user declines the any-version fallback, a recorded pin is + left uninstalled — so the install must FAIL, not silently succeed (P0-1). + Previously this returned True, yielding "Environment ready" + a dead run.""" venv_dir = tmp_path / ".venv" venv_dir.mkdir() repo_dir = tmp_path @@ -716,7 +718,7 @@ def test_skips_failed_packages_when_user_declines_fallback(self, service, tmp_pa auto_confirm=False, ) - assert success is True + assert success is False assert any("exact version not found" in w for w in warnings) def test_identifies_individual_failed_packages(self, service, tmp_path): diff --git a/tests/unit/test_reproduce_pin_failure.py b/tests/unit/test_reproduce_pin_failure.py new file mode 100644 index 00000000..85199a48 --- /dev/null +++ b/tests/unit/test_reproduce_pin_failure.py @@ -0,0 +1,98 @@ +"""P0-1: reproduce must FAIL (not report success) when a recorded pip pin can't +install, and must offer a debuggable export of the pins.""" + +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +from roar.execution.reproduction.installers import PythonPackageInstaller + + +def _installer() -> PythonPackageInstaller: + return PythonPackageInstaller(use_uv=False, print_fn=lambda *_: None) + + +def _pip(rc: int, stderr: str = "") -> SimpleNamespace: + return SimpleNamespace(returncode=rc, stderr=stderr, stdout="") + + +def _fake_run_pip(*, fail_exact: bool = True, fail_fallback: bool = False, all_ok: bool = False): + def run_pip(venv_dir, repo_dir, args, show_output=False): + if all_ok: + return _pip(0) + # Exact-pin installs carry "==" and the per-package probe carries "--dry-run"; + # the recovery install of an *unversioned* name is the fallback. + if any("==" in a for a in args) or "--dry-run" in args: + return _pip(1 if fail_exact else 0, "No matching distribution") + return _pip(1 if fail_fallback else 0, "No matching distribution") + + return run_pip + + +def test_returns_false_when_pin_unresolvable_and_no_any_version(monkeypatch): + inst = _installer() + monkeypatch.setattr(inst, "_run_pip", _fake_run_pip(fail_exact=True)) + ok, _warnings = inst.install_packages( + Path("/venv"), + ["yanked-pkg==9.9.9"], + Path("/repo"), + auto_confirm=True, + allow_any_version=False, + ) + assert ok is False # was True before the fix -> "Environment ready" + dead run + + +def test_returns_false_when_any_version_fallback_also_fails(monkeypatch): + inst = _installer() + monkeypatch.setattr(inst, "_run_pip", _fake_run_pip(fail_exact=True, fail_fallback=True)) + ok, _warnings = inst.install_packages( + Path("/venv"), + ["private-pkg==1.0"], + Path("/repo"), + auto_confirm=True, + allow_any_version=True, + ) + assert ok is False + + +def test_returns_true_when_all_pins_install(monkeypatch): + inst = _installer() + monkeypatch.setattr(inst, "_run_pip", _fake_run_pip(all_ok=True)) + ok, warnings = inst.install_packages( + Path("/venv"), ["numpy==2.0.0"], Path("/repo"), auto_confirm=True + ) + assert ok is True + assert warnings == [] + + +def test_any_version_recovery_returns_true_with_warning(monkeypatch): + inst = _installer() + monkeypatch.setattr(inst, "_run_pip", _fake_run_pip(fail_exact=True, fail_fallback=False)) + ok, warnings = inst.install_packages( + Path("/venv"), + ["driftable-pkg==1.0"], + Path("/repo"), + auto_confirm=True, + allow_any_version=True, # the bypass: install an available version + ) + assert ok is True + assert any("driftable-pkg" in w for w in warnings) + + +def test_export_requirements_writes_recorded_pins(tmp_path): + from roar.application.reproduce.service import _export_pip_requirements + + pipeline = SimpleNamespace( + build_steps=[], + run_steps=[{"metadata": {"packages": {"pip": {"numpy": "2.0.0", "torch": "2.7.0"}}}}], + artifact_hash="abc123def456", + session_hash=None, + ) + out = MagicMock() + dest = tmp_path / "req.txt" + _export_pip_requirements(pipeline, str(dest), out) + + text = dest.read_text() + assert "numpy==2.0.0" in text + assert "torch==2.7.0" in text + assert text.lstrip().startswith("#") # has the debug header From 3a75b0f87b5cffcb378f02d8dbb80e6a18e56123 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Thu, 6 Aug 2026 13:18:28 +0000 Subject: [PATCH 04/52] =?UTF-8?q?test(reproduce):=20P0-2=20=E2=80=94=20def?= =?UTF-8?q?ault=20no=20timeout=20+=20timeout=20kills=20the=20whole=20proce?= =?UTF-8?q?ss=20group?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also use contextlib.suppress for the post-kill reap (ruff). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../reproduction/pipeline_executor.py | 5 +- tests/unit/test_pipeline_executor_timeout.py | 73 +++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_pipeline_executor_timeout.py diff --git a/roar/execution/reproduction/pipeline_executor.py b/roar/execution/reproduction/pipeline_executor.py index eba87893..5a82d7b2 100644 --- a/roar/execution/reproduction/pipeline_executor.py +++ b/roar/execution/reproduction/pipeline_executor.py @@ -5,6 +5,7 @@ This service handles executing pipeline steps during reproduction. """ +import contextlib import json import os import shutil @@ -208,10 +209,8 @@ def _kill_process_group(proc: "subprocess.Popen[bytes]") -> None: os.killpg(os.getpgid(proc.pid), signal.SIGKILL) except (ProcessLookupError, PermissionError): proc.kill() - try: + with contextlib.suppress(subprocess.TimeoutExpired): proc.wait(timeout=30) - except subprocess.TimeoutExpired: - pass def _wrap_with_roar( self, diff --git a/tests/unit/test_pipeline_executor_timeout.py b/tests/unit/test_pipeline_executor_timeout.py new file mode 100644 index 00000000..42dce49a --- /dev/null +++ b/tests/unit/test_pipeline_executor_timeout.py @@ -0,0 +1,73 @@ +"""P0-2: per-step timeout is configurable (default none) and, when it fires, +kills the whole process group — not just the shell — so a grandchild workload +(e.g. train.py) can't keep running past the declared failure.""" + +import os +import sys +import time +from unittest.mock import MagicMock + +from roar.execution.reproduction.pipeline_executor import PipelineExecutor + + +def _executor(step_timeout=None): + ex = PipelineExecutor(roar_executable="/bin/true", step_timeout=step_timeout) + ex._print = lambda *_: None + return ex + + +def _drive(ex, wrapped_command, environment): + """Run one step, forcing the wrapped command and a clean env.""" + ex._wrap_with_roar = lambda *a, **k: wrapped_command + ex._prepare_environment = lambda *a, **k: dict(os.environ) + step = {"command": "x", "metadata": {}} + return ex._run_step(step, environment, is_build=False) + + +def test_default_step_timeout_is_none(): + assert PipelineExecutor()._step_timeout is None + + +def test_quick_command_succeeds_with_no_timeout(tmp_path): + ex = _executor(step_timeout=None) + env = MagicMock(repo_dir=tmp_path) + assert _drive(ex, f'{sys.executable} -c "pass"', env) is True + + +def test_failing_command_returns_false(tmp_path): + ex = _executor(step_timeout=None) + env = MagicMock(repo_dir=tmp_path) + assert _drive(ex, f'{sys.executable} -c "raise SystemExit(3)"', env) is False + + +def test_timeout_kills_the_whole_process_group(tmp_path): + """A shell child that spawns a long-lived grandchild must be fully reaped on + timeout. Before the fix (shell=True + subprocess.run timeout), only the shell + died and the grandchild ran on. The grandchild here touches a marker after a + long sleep; if the tree was killed the marker must NOT appear.""" + marker = tmp_path / "grandchild_finished" + child = tmp_path / "child.py" + child.write_text( + "import subprocess, sys, time\n" + "grand = (\n" + " 'import time; time.sleep(20); " + f"open({str(marker)!r}, \"w\").close()'\n" + ")\n" + "subprocess.Popen([sys.executable, '-c', grand])\n" + "time.sleep(20)\n" + ) + + ex = _executor(step_timeout=1) + env = MagicMock(repo_dir=tmp_path) + + start = time.monotonic() + result = _drive(ex, f"{sys.executable} {child}", env) + elapsed = time.monotonic() - start + + assert result is False + assert elapsed < 8, f"timeout should fire ~1s, took {elapsed:.1f}s" + + # Give any survivor time to reach its marker write; it must not, because the + # whole process group (incl. the grandchild) was SIGKILLed. + time.sleep(3) + assert not marker.exists(), "grandchild survived the timeout — process group not killed" From cf203d795d1c476a3d370034e83e6e57ce43cab5 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Thu, 6 Aug 2026 13:45:51 +0000 Subject: [PATCH 05/52] test: make process-group-kill test fast (1.5s) and actually exercise the grandchild MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prior version's grandchild ran via a multi-line `python -c` payload whose single-quoted literal spanned physical lines — a SyntaxError, so the grandchild never started and the assertion was silently inconclusive. It also blocked on a fixed 3s sleep, adding dead wall-clock to the slow macOS lane. Rewrite: grandchild is a real script that heartbeats a counter file every 50ms. After the step times out we assert the counter is frozen (killed) rather than waiting out a sleep, and a `heartbeat.exists()` guard proves the grandchild actually started. ~3.0s -> ~1.5s, stable, and now a real test. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/unit/test_pipeline_executor_timeout.py | 51 +++++++++++++++----- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/tests/unit/test_pipeline_executor_timeout.py b/tests/unit/test_pipeline_executor_timeout.py index 42dce49a..96dd1b44 100644 --- a/tests/unit/test_pipeline_executor_timeout.py +++ b/tests/unit/test_pipeline_executor_timeout.py @@ -43,18 +43,32 @@ def test_failing_command_returns_false(tmp_path): def test_timeout_kills_the_whole_process_group(tmp_path): """A shell child that spawns a long-lived grandchild must be fully reaped on timeout. Before the fix (shell=True + subprocess.run timeout), only the shell - died and the grandchild ran on. The grandchild here touches a marker after a - long sleep; if the tree was killed the marker must NOT appear.""" - marker = tmp_path / "grandchild_finished" + died and the grandchild ran on — a false failure plus a silent GPU-cost leak. + + We prove the kill by *liveness*, not by waiting out a sleep: the grandchild + bumps a counter file every 50ms. Once the step times out and the group is + SIGKILLed, the counter must stop advancing. (A plain ``os.kill(pid, 0)`` + check is unreliable here — a killed-but-unreaped grandchild is a zombie, for + which ``os.kill`` still reports "alive".) No long fixed sleep, so this stays + ~1.5s on the slow macOS lane.""" + heartbeat = tmp_path / "grandchild.heartbeat" + # Real files, not `python -c` payloads — the -c escaping for a multi-line + # loop is a trap (a single broken literal makes the child a no-op and the + # test silently inconclusive). + grand = tmp_path / "grand.py" + grand.write_text( + "import time\n" + "i = 0\n" + "while True:\n" + f" open({str(heartbeat)!r}, 'w').write(str(i))\n" + " i += 1\n" + " time.sleep(0.05)\n" + ) child = tmp_path / "child.py" child.write_text( "import subprocess, sys, time\n" - "grand = (\n" - " 'import time; time.sleep(20); " - f"open({str(marker)!r}, \"w\").close()'\n" - ")\n" - "subprocess.Popen([sys.executable, '-c', grand])\n" - "time.sleep(20)\n" + f"subprocess.Popen([sys.executable, {str(grand)!r}])\n" + "time.sleep(30)\n" ) ex = _executor(step_timeout=1) @@ -67,7 +81,18 @@ def test_timeout_kills_the_whole_process_group(tmp_path): assert result is False assert elapsed < 8, f"timeout should fire ~1s, took {elapsed:.1f}s" - # Give any survivor time to reach its marker write; it must not, because the - # whole process group (incl. the grandchild) was SIGKILLed. - time.sleep(3) - assert not marker.exists(), "grandchild survived the timeout — process group not killed" + # The grandchild starts heartbeating well within the 1s timeout. + deadline = start + 3 + while not heartbeat.exists() and time.monotonic() < deadline: + time.sleep(0.05) + assert heartbeat.exists(), "grandchild never started — test inconclusive" + + # If the whole group was killed the counter is frozen; a survivor keeps + # advancing it. Two reads 0.5s apart (>> the 50ms heartbeat) settle it. + before = heartbeat.read_text() + time.sleep(0.5) + after = heartbeat.read_text() + assert before == after, ( + f"grandchild kept running after the timeout ({before!r} -> {after!r}) " + "— process group not killed" + ) From 86c9a10d3cda3061e49e52c39dda371449791766 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Thu, 6 Aug 2026 13:59:06 +0000 Subject: [PATCH 06/52] ci(macos): run only the tracer / platform-dependent test subset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The macOS job ran the entire default suite (~1180 tests) — the same OS-independent pure-Python logic the Linux `test` job already covers across 5 Python versions. On slow macOS runners that adds wall-clock and intermittent job timeouts without adding signal: the only thing macOS uniquely exercises is the OS-specific surface (the DYLD preload tracer, sitecustomize/runtime injection, the native `_hash_native` extension, and the real `roar run` product path). Scope the macOS job to those platform-dependent trees, selected by PATH rather than by marker: tests/execution/runtime tests/integration tests/happy_path tests/application/run tests/backends/local/integration plus two hashing-value files (test_hashing_backend, test_canonical_session_hash) so a macOS-specific _hash_native ABI/endianness regression still can't slip through. Path selection is deliberate over a `macos` marker: with --strict-markers and 100+ platform-relevant files, per-file tagging is high-churn and easy to under-apply — which is exactly how a macOS-only tracer test would be silently dropped. Directory selection fails safe: the Linux-only tracer regressions that live inside these dirs already carry skipif(platform != "Linux") and simply skip on macOS. Collected macOS tests drop from ~2355 to 286 (~88% fewer) with no reduction in platform coverage. The macOS-only preload smoke step and the native-hash import verify step are unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 235e801c..2625d65d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -272,11 +272,32 @@ jobs: python -m roar run "$(python -c 'import sys; print(sys.executable)')" smoke.py - - name: Run tests (parallel) + - name: Run tracer / platform-dependent tests (parallel) + # macOS CI exists to cover the OS-specific surface: the DYLD preload + # tracer, sitecustomize/runtime injection, the native _hash_native + # extension, and the real `roar run` product path. The rest of the suite + # is OS-independent pure-Python logic already fully covered by the Linux + # `test` job (5 Python versions), so running all ~1180 tests here only + # adds slow-runner wall-clock (and timeouts) without adding signal. + # + # Select the platform-dependent trees by path rather than by marker: + # low-churn, self-documenting, and fails safe — Linux-only tracer + # regressions living inside these dirs carry their own + # skipif(platform != "Linux") and simply skip on macOS, and there is no + # per-file marker to forget to apply (which is how a macOS-only tracer + # test would otherwise be silently dropped). The two hashing-value files + # are kept so a macOS-specific _hash_native ABI/endianness regression + # still can't slip through, even though the hashing *logic* is covered + # on Linux. run: > pytest --tb=short -x -m "not glaas and not live_glaas and not ebpf and not large_pipeline" - --ignore=tests/backends/osmo - --ignore-glob=tests/backends/test_osmo*.py + tests/execution/runtime + tests/integration + tests/happy_path + tests/application/run + tests/backends/local/integration + tests/unit/test_hashing_backend.py + tests/unit/test_canonical_session_hash.py --ignore=tests/execution/runtime/test_sitecustomize_perf.py test-tracer-privileged: From 822639454d7b375b79a64e48bbb417e148169c47 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Thu, 6 Aug 2026 14:15:14 +0000 Subject: [PATCH 07/52] style: apply ruff format to pipeline_executor timeout message CI runs `ruff format --check .` in addition to `ruff check`; the timeout message fits on one line under the limit. No behavior change. Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/execution/reproduction/pipeline_executor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roar/execution/reproduction/pipeline_executor.py b/roar/execution/reproduction/pipeline_executor.py index 5a82d7b2..ad48a8fb 100644 --- a/roar/execution/reproduction/pipeline_executor.py +++ b/roar/execution/reproduction/pipeline_executor.py @@ -179,8 +179,7 @@ def _run_step( returncode = proc.wait(timeout=self._step_timeout) except subprocess.TimeoutExpired: self._print( - f" Step timed out after {self._step_timeout}s — " - "killing the process group" + f" Step timed out after {self._step_timeout}s — killing the process group" ) self._kill_process_group(proc) return False From 473dc7bb967e8dcc50666c15c445464862093a8e Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Thu, 6 Aug 2026 17:16:34 +0000 Subject: [PATCH 08/52] fix: catch the recovery-path false-green (swallowed combined-install RC) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The batch-install-failed recovery path re-installs the individually-resolvable pins together but discarded that install's return code. Because each pin was probed with a per-package `pip install --dry-run` (resolvable ALONE) rather than jointly, a set that resolves individually but conflicts in combination produced an empty `failed_packages`, so the honest-fail guard never fired: pip left an incomplete venv (MiniMind-O: 35 of 105 packages) and reproduce still printed "Environment ready". Capture the combined install's return code; on non-zero, treat those pins as unresolved (they are not jointly installable) and fail honestly with a message that names the conflict. `unresolved_packages` is now accumulated with extend() so the failed-pin branches don't clobber a combined-conflict result. Scope note: this closes the observed false-green but does NOT make env-setup bulletproof — the robust fix is to verify the rebuilt venv against the recorded package set (post-install `pip freeze`/`pip check` diff) rather than trusting install return codes at all. That verification redesign is tracked as a separate roar-core P0. Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/execution/reproduction/installers.py | 23 +++++++++++++++++--- tests/unit/test_reproduce_pin_failure.py | 26 +++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/roar/execution/reproduction/installers.py b/roar/execution/reproduction/installers.py index 5a4e5c49..38aeb30b 100644 --- a/roar/execution/reproduction/installers.py +++ b/roar/execution/reproduction/installers.py @@ -254,6 +254,7 @@ def install_packages( ) -> tuple[bool, list[str]]: warnings: list[str] = [] unresolved_packages: list[str] = [] + conflict_in_combination = False active_presenter = presenter or self._presenter if not packages: self._print("No packages to install from provenance.") @@ -278,7 +279,18 @@ def install_packages( succeeded_packages.append(package) if succeeded_packages: - self._run_pip(venv_dir, repo_dir, ["install", *succeeded_packages], show_output=True) + combined = self._run_pip( + venv_dir, repo_dir, ["install", *succeeded_packages], show_output=True + ) + if combined.returncode != 0: + # Each of these pins resolved on its own (the per-package --dry-run + # above), but they conflict in combination: pip exited non-zero and + # left an incomplete/inconsistent venv. Individually-resolvable is + # NOT jointly-installable, so treat them as unresolved rather than + # discarding this return code and printing a false "Environment + # ready" over a venv missing most of its packages. + conflict_in_combination = True + unresolved_packages.extend(succeeded_packages) if failed_packages: self._print(f"\nExact versions not found for {len(failed_packages)} pip packages:") @@ -303,7 +315,7 @@ def install_packages( warnings.append( f"Some pip packages failed to install: {(fallback.stderr or '').strip()}" ) - unresolved_packages = list(failed_packages) + unresolved_packages.extend(failed_packages) else: for package in failed_packages: warnings.append( @@ -312,7 +324,7 @@ def install_packages( else: for package in failed_packages: warnings.append(f"Skipped {package} (exact version not found)") - unresolved_packages = list(failed_packages) + unresolved_packages.extend(failed_packages) if unresolved_packages: # A recorded pin could not be installed. Do NOT report success — that @@ -325,6 +337,11 @@ def install_packages( ) for package in unresolved_packages: self._print(f" - {package}") + if conflict_in_combination: + self._print( + "These pins resolve individually but conflict when installed " + "together, so pip left an incomplete environment." + ) self._print( "Re-run with --pip-any-version to install available versions instead, " "or --export-requirements to inspect/try the exact pins yourself." diff --git a/tests/unit/test_reproduce_pin_failure.py b/tests/unit/test_reproduce_pin_failure.py index 85199a48..3e12cfd3 100644 --- a/tests/unit/test_reproduce_pin_failure.py +++ b/tests/unit/test_reproduce_pin_failure.py @@ -55,6 +55,32 @@ def test_returns_false_when_any_version_fallback_also_fails(monkeypatch): assert ok is False +def test_returns_false_when_individually_resolvable_pins_conflict_in_combination(monkeypatch): + """The recovery path: the batch install fails, every pin passes its per-package + --dry-run (resolvable ALONE), but the combined re-install conflicts and pip + leaves an incomplete venv. Before the guard, that combined install's return + code was discarded -> "Environment ready" over a venv missing most packages + (MiniMind-O: 35 of 105). Must now fail honestly.""" + inst = _installer() + + def run_pip(venv_dir, repo_dir, args, show_output=False): + if "--dry-run" in args: + return _pip(0) # each pin resolves on its own + if any("==" in a for a in args): + return _pip(1, "ResolutionImpossible") # ...but not together + return _pip(0) + + monkeypatch.setattr(inst, "_run_pip", run_pip) + ok, _warnings = inst.install_packages( + Path("/venv"), + ["torch==2.7.0", "numpy==2.0.0"], + Path("/repo"), + auto_confirm=True, + allow_any_version=False, + ) + assert ok is False + + def test_returns_true_when_all_pins_install(monkeypatch): inst = _installer() monkeypatch.setattr(inst, "_run_pip", _fake_run_pip(all_ok=True)) From 5a774f6e086d7b1896dff9e60df0acf97f9018a4 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Thu, 6 Aug 2026 21:29:04 +0000 Subject: [PATCH 09/52] reproduce: warn + confirm on a Python major.minor mismatch (P0-4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without uv, `roar reproduce` can only build the venv with the interpreter roar itself runs under. When that differs from the recorded interpreter at the major.minor level it used to build the wrong-version venv silently, warn once, and continue — then the recorded (e.g. cp312) wheels fail to install and the error blamed the package list. Now, on a major.minor mismatch we: - warn loudly (recorded vs building), explaining the ABI-tag risk, - recommend uv and link its install docs (uv provisions the EXACT recorded interpreter — the deterministic fix; no PATH-guessing among multiple pythons), - ask "continue anyway?" (default no); `-y/--yes` overrides to continue. Declining aborts the reproduction rather than silently using the wrong Python. Patch-level differences (3.12.9 vs 3.12.10) still pass silently — not reproducibility-relevant. Many pure-Python repos reproduce fine on a different minor, which is why this warns-and-asks rather than hard-failing. `auto_confirm` is threaded setup -> setup_in_place -> _create_venv -> _create_venv_uv; the CLI already maps `-y/--yes` to it. Tests: tests/integration/test_reproduce_python_mismatch.py builds REAL venvs (no subprocess mock) to exercise the whole non-uv path end-to-end — declined aborts (+ uv link shown), --yes overrides and builds, confirmed continues, matching-minor stays silent. Existing env-setup tests updated to the new prompt/abort contract. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../reproduction/environment_setup.py | 83 +++++++++++++++---- .../test_reproduce_python_mismatch.py | 78 +++++++++++++++++ tests/unit/test_environment_setup.py | 25 +++--- 3 files changed, 161 insertions(+), 25 deletions(-) create mode 100644 tests/integration/test_reproduce_python_mismatch.py diff --git a/roar/execution/reproduction/environment_setup.py b/roar/execution/reproduction/environment_setup.py index b47cef9d..438a3958 100644 --- a/roar/execution/reproduction/environment_setup.py +++ b/roar/execution/reproduction/environment_setup.py @@ -149,7 +149,9 @@ def setup_in_place( # Create virtual environment, pinned to the recorded interpreter. self.logger.debug("Creating virtual environment...") - venv_dir = self._create_venv(repo_dir, self._recorded_python_version(pipeline)) + venv_dir = self._create_venv( + repo_dir, self._recorded_python_version(pipeline), auto_confirm=auto_confirm + ) self.logger.debug("Virtual environment created at: %s", venv_dir) # Initialize roar in the cloned repository @@ -458,14 +460,19 @@ def _clone_repository( return repo_dir - def _create_venv(self, repo_dir: Path, target_version: str | None = None) -> Path: + def _create_venv( + self, repo_dir: Path, target_version: str | None = None, auto_confirm: bool = False + ) -> Path: """ Create virtual environment in repository, pinned to the recorded Python. - We try the exact recorded interpreter (uv downloads a managed build if - needed), then the recorded major.minor, then fall back to the default - with a warning. We never block — a different interpreter still - reproduces, just less faithfully ("same setup" is best-effort). + With uv we provision the *exact* recorded interpreter (uv downloads a + managed build if needed). Without uv we can only use the interpreter roar + is running under; if that differs from the recorded one at the major.minor + level we warn, recommend uv, and — unless ``auto_confirm`` — ask before + continuing, because the recorded packages may not install or behave the + same (see :meth:`_confirm_python_mismatch`). We do not silently substitute + a different interpreter and then blame the package list. Returns: Path to venv directory @@ -479,7 +486,7 @@ def _create_venv(self, repo_dir: Path, target_version: str | None = None) -> Pat self._print("Creating virtual environment...") if self._use_uv: - self._create_venv_uv(venv_dir, repo_dir, target_version) + self._create_venv_uv(venv_dir, repo_dir, target_version, auto_confirm) else: # `python -m venv` can only use the running interpreter. subprocess.run( @@ -487,7 +494,7 @@ def _create_venv(self, repo_dir: Path, target_version: str | None = None) -> Pat check=True, cwd=repo_dir, ) - self._warn_python_mismatch(target_version, self._get_python_version()) + self._confirm_python_mismatch(target_version, self._get_python_version(), auto_confirm) gitignore = venv_dir / ".gitignore" if not gitignore.exists(): @@ -495,7 +502,13 @@ def _create_venv(self, repo_dir: Path, target_version: str | None = None) -> Pat return venv_dir - def _create_venv_uv(self, venv_dir: Path, repo_dir: Path, target_version: str | None) -> None: + def _create_venv_uv( + self, + venv_dir: Path, + repo_dir: Path, + target_version: str | None, + auto_confirm: bool = False, + ) -> None: """Create the venv with uv, pinned to the recorded interpreter if we can.""" for version in self._python_candidates(target_version): result = subprocess.run( @@ -513,7 +526,9 @@ def _create_venv_uv(self, venv_dir: Path, repo_dir: Path, target_version: str | # Couldn't provision the recorded interpreter — use uv's default, then warn. subprocess.run(["uv", "venv", str(venv_dir)], check=True, cwd=repo_dir) - self._warn_python_mismatch(target_version, self._venv_python_version(venv_dir)) + self._confirm_python_mismatch( + target_version, self._venv_python_version(venv_dir), auto_confirm + ) def _recorded_python_version(self, pipeline: "PipelineInfo") -> str | None: """The interpreter version recorded for this lineage (e.g. '3.14.4'), or None.""" @@ -535,21 +550,59 @@ def _python_candidates(target_version: str | None) -> list[str]: candidates.append(minor) return candidates - def _warn_python_mismatch(self, recorded: str | None, actual: str | None) -> None: - """Warn when the venv's interpreter differs from the recorded one at the - major.minor level. Patch differences (3.14.4 vs 3.14.6) aren't - reproducibility-relevant, so they don't warn.""" + def _confirm_python_mismatch( + self, recorded: str | None, actual: str | None, auto_confirm: bool + ) -> None: + """Handle a major.minor interpreter mismatch between capture and reproduce. + + Patch-level differences (3.14.4 vs 3.14.6) aren't reproducibility-relevant + and pass silently. For a major.minor mismatch the recorded packages (e.g. + ABI-tagged wheels) may fail to install or behave differently, so we warn + loudly, recommend uv (which provisions the *exact* recorded interpreter), + and — unless ``auto_confirm`` (``--yes``) — ask before continuing. + Declining aborts the reproduction rather than silently using the wrong + Python. + + Many pure-Python repos still reproduce fine on a different minor, which is + why this warns-and-asks rather than hard-failing. + """ if not recorded: return rec_minor = ".".join(recorded.split(".")[:2]) act_minor = ".".join((actual or "").split(".")[:2]) if act_minor and act_minor == rec_minor: return + using = actual or "a different interpreter" + uv_url = "https://docs.astral.sh/uv/getting-started/installation/" + bar = "=" * 64 self._print( - f"⚠ Recorded Python was {recorded}; reproducing with {using} — results may differ." + f"\n{bar}\n" + "⚠ PYTHON VERSION MISMATCH\n" + f" Recorded at capture: Python {recorded}\n" + f" Reproducing with: Python {using}\n" + f" The recorded packages were built for {rec_minor}; some (e.g.\n" + " ABI-tagged wheels) may fail to install or behave differently on " + f"{act_minor or 'this interpreter'}.\n\n" + " For a faithful reproduction, install uv and re-run — roar will then\n" + " provision the exact recorded interpreter automatically:\n" + f" {uv_url}\n" + f"{bar}" ) + if auto_confirm: + self._print("Continuing with the mismatched interpreter (--yes).") + return + + if not self._presenter.confirm( + f"Continue reproducing with Python {using} anyway?", default=False + ): + raise RuntimeError( + f"Reproduction aborted: recorded Python {recorded} is not available " + f"(no uv to provision it). Install uv ({uv_url}) or re-run with " + "--yes to proceed anyway." + ) + @staticmethod def _venv_python_version(venv_dir: Path) -> str | None: """Read the created venv's Python version from pyvenv.cfg, or None.""" diff --git a/tests/integration/test_reproduce_python_mismatch.py b/tests/integration/test_reproduce_python_mismatch.py new file mode 100644 index 00000000..465d18bb --- /dev/null +++ b/tests/integration/test_reproduce_python_mismatch.py @@ -0,0 +1,78 @@ +"""P0-4 end-to-end: when reproduce cannot provision the recorded Python (no uv) +and the running interpreter differs at major.minor, roar warns loudly, recommends +uv, and asks before continuing — with ``--yes`` (auto_confirm) overriding the +prompt. Declining aborts instead of silently reproducing on the wrong Python. + +These build a REAL venv via ``python -m venv`` (no subprocess mock) so the whole +non-uv path is exercised, including the actual interpreter-version comparison. +""" + +import sys +from unittest.mock import MagicMock + +import pytest + +from roar.execution.reproduction.environment_setup import EnvironmentSetupService + + +def _running_minor() -> str: + return f"{sys.version_info.major}.{sys.version_info.minor}" + + +def _svc(confirm_return: bool | None = None): + presenter = MagicMock() + if confirm_return is not None: + presenter.confirm.return_value = confirm_return + svc = EnvironmentSetupService(presenter=presenter) + svc._use_uv = False # force the `python -m venv` (running-interpreter) path + return svc, presenter + + +def _printed(presenter) -> str: + return " ".join(str(c.args[0]) for c in presenter.print.call_args_list) + + +def test_mismatch_declined_aborts(tmp_path): + """No --yes, user declines the prompt -> RuntimeError, and the warning both + names the mismatch and points at the uv install docs.""" + svc, presenter = _svc(confirm_return=False) + repo = tmp_path / "repo" + repo.mkdir() + with pytest.raises(RuntimeError, match="aborted"): + svc._create_venv(repo, "3.99.0", auto_confirm=False) # 3.99 can't match the runner + out = _printed(presenter) + assert "PYTHON VERSION MISMATCH" in out + assert "docs.astral.sh/uv" in out + presenter.confirm.assert_called_once() + + +def test_mismatch_yes_overrides_prompt_and_builds_venv(tmp_path): + """--yes -> warn loudly but continue without asking; a real venv is built.""" + svc, presenter = _svc() + repo = tmp_path / "repo" + repo.mkdir() + venv = svc._create_venv(repo, "3.99.0", auto_confirm=True) + assert (venv / "pyvenv.cfg").exists() # a genuine venv was created + presenter.confirm.assert_not_called() # --yes means no prompt + assert "PYTHON VERSION MISMATCH" in _printed(presenter) + + +def test_mismatch_confirmed_continues(tmp_path): + """No --yes, user accepts -> continue and build the venv.""" + svc, presenter = _svc(confirm_return=True) + repo = tmp_path / "repo" + repo.mkdir() + venv = svc._create_venv(repo, "3.99.0", auto_confirm=False) + assert (venv / "pyvenv.cfg").exists() + presenter.confirm.assert_called_once() + + +def test_matching_minor_no_prompt_no_warning(tmp_path): + """Recorded minor == running minor -> silent: no warning, no prompt, venv built.""" + svc, presenter = _svc(confirm_return=False) + repo = tmp_path / "repo" + repo.mkdir() + venv = svc._create_venv(repo, f"{_running_minor()}.0", auto_confirm=False) + assert (venv / "pyvenv.cfg").exists() + presenter.confirm.assert_not_called() + assert "MISMATCH" not in _printed(presenter) diff --git a/tests/unit/test_environment_setup.py b/tests/unit/test_environment_setup.py index 63016926..02402ccd 100644 --- a/tests/unit/test_environment_setup.py +++ b/tests/unit/test_environment_setup.py @@ -806,14 +806,17 @@ def test_recorded_version_none_when_absent(self): pipeline.run_steps = [{"metadata": json.dumps({"packages": {}})}] assert svc._recorded_python_version(pipeline) is None - def test_warn_only_on_minor_mismatch(self): + def test_confirm_only_warns_on_minor_mismatch(self): svc = self._svc() - svc._warn_python_mismatch("3.14.4", "3.14.9") # same minor -> no warn - svc._warn_python_mismatch(None, "3.13.0") # nothing recorded -> no warn + # auto_confirm=True so a mismatch warns-and-continues (no prompt/abort). + svc._confirm_python_mismatch("3.14.4", "3.14.9", auto_confirm=True) # same minor + svc._confirm_python_mismatch(None, "3.13.0", auto_confirm=True) # nothing recorded assert svc._presenter.print.call_count == 0 - svc._warn_python_mismatch("3.14.4", "3.13.14") # minor differs -> warn - msg = svc._presenter.print.call_args[0][0] - assert "Recorded Python was 3.14.4" in msg and "3.13.14" in msg + svc._confirm_python_mismatch("3.14.4", "3.13.14", auto_confirm=True) # minor differs + printed = " ".join(str(c.args[0]) for c in svc._presenter.print.call_args_list) + assert "PYTHON VERSION MISMATCH" in printed + assert "3.14.4" in printed and "3.13.14" in printed + assert "uv" in printed # recommends the deterministic fix def test_uv_venv_pins_recorded_version(self, tmp_path): svc = self._svc() @@ -833,7 +836,7 @@ def fake_run(cmd, **kwargs): args = run.call_args_list[0].args[0] assert args[:2] == ["uv", "venv"] and "--python" in args and "3.14.4" in args # matched the recorded interpreter -> no mismatch warning - assert not any("Recorded Python" in str(c) for c in svc._presenter.print.call_args_list) + assert not any("MISMATCH" in str(c) for c in svc._presenter.print.call_args_list) def test_uv_falls_back_to_minor_then_default_with_warning(self, tmp_path): svc = self._svc() @@ -854,11 +857,13 @@ def fake_run(cmd, **kwargs): return MagicMock(returncode=0) with patch("subprocess.run", side_effect=fake_run): - svc._create_venv(repo_dir, "3.14.4") + # auto_confirm so the resulting mismatch (3.14.4 -> 3.13.14) doesn't prompt. + svc._create_venv(repo_dir, "3.14.4", auto_confirm=True) pythons = [c for c in calls if "--python" in c] assert any("3.14.4" in c for c in pythons) assert any("3.14" in c and "3.14.4" not in c for c in pythons) assert calls[-1] == ["uv", "venv", str(venv_dir)] # bare fallback last - warning = svc._presenter.print.call_args[0][0] - assert "Recorded Python was 3.14.4" in warning and "3.13.14" in warning + printed = " ".join(str(c.args[0]) for c in svc._presenter.print.call_args_list) + assert "PYTHON VERSION MISMATCH" in printed + assert "3.14.4" in printed and "3.13.14" in printed From e558fd270fbd1c03fecf521f344404b5918add2d Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Thu, 6 Aug 2026 22:24:32 +0000 Subject: [PATCH 10/52] tracker: per-PID inject-log shards + union merge (P0-9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every process in a traced tree inherits the same ROAR_LOG_FILE, and write_log() opened it "w" — so each process truncated the others and the surviving record was whichever wrote LAST. With multiprocessing (litdata/DataLoader workers, HF datasets num_proc, torchrun ranks) that was often a worker with a subset of the imports — or none. Same command, different record: 45 packages (parent) or 10 (a litdata worker), decided only by scheduling. This is distinct from #264 (which changes how packages are derived FROM a record, not WHICH record survives), so it needs its own fix. write_log now writes a per-PID shard (`{ROAR_LOG_FILE}.`), and the tracer unions the shards into the canonical inject log after the run (merge_inject_logs): set/dict activity (opened_files, imported_modules, modules_files, used_packages, installed_packages, ...) is unioned across the tree; scalar identity (argv, python_version, ...) is taken from the richest shard — the workload imports everything, a `python -c` worker a subset. The collector reads the merged file unchanged; cleanup sweeps stray shards. Tests (tests/execution/runtime/test_inject_log_merge.py): write_log writes a shard not the shared file; a sparse `['-c']` worker shard does not clobber the workload's argv or packages (the exact litdata parent/worker shape); concrete versions beat None on union; no-op without shards. Existing tracker tests updated to merge the shard before asserting. Note: full multi-process `roar run` confirmation requires the built tracer binaries (not present in an editable checkout); the merge is covered here with faithful shards and confirmed end-to-end by the row-009 re-certification. Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/execution/runtime/coordinator.py | 8 ++ roar/execution/runtime/inject/tracker.py | 68 ++++++++++- roar/execution/runtime/tracer.py | 8 ++ .../runtime/test_inject_log_merge.py | 110 ++++++++++++++++++ .../execution/runtime/test_runtime_tracker.py | 4 + 5 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 tests/execution/runtime/test_inject_log_merge.py diff --git a/roar/execution/runtime/coordinator.py b/roar/execution/runtime/coordinator.py index 8450b48a..a5afccb0 100644 --- a/roar/execution/runtime/coordinator.py +++ b/roar/execution/runtime/coordinator.py @@ -7,6 +7,8 @@ from __future__ import annotations +import contextlib +import glob import os import secrets import sys @@ -467,3 +469,9 @@ def _cleanup_logs(self, tracer_log: str, inject_log: str) -> None: os.remove(log_file) except OSError: pass + # Sweep any per-PID inject-log shards that merge_inject_logs didn't reach + # (e.g. a report written after the merge, or a merge that never ran). + if inject_log: + for shard in glob.glob(glob.escape(inject_log) + ".*"): + with contextlib.suppress(OSError): + os.remove(shard) diff --git a/roar/execution/runtime/inject/tracker.py b/roar/execution/runtime/inject/tracker.py index 24d067df..2a06a62d 100644 --- a/roar/execution/runtime/inject/tracker.py +++ b/roar/execution/runtime/inject/tracker.py @@ -4,6 +4,7 @@ import builtins import contextlib +import glob import json import os import platform @@ -106,6 +107,61 @@ def get_used_packages( return used +_MERGE_LIST_FIELDS = ("opened_files", "imported_modules", "modules_files", "shared_libs") +_MERGE_DICT_FIELDS = ("used_packages", "installed_packages", "env_reads") + + +def merge_inject_logs(base_path: str) -> None: + """Union per-PID inject-log shards (``{base_path}.``) into one record at + ``base_path``. + + Every process in a traced tree writes its own shard (see + :meth:`RuntimeInjectionTracker.write_log`). Unioning them recovers the full + workload — packages, files and imports seen by the parent AND by any worker — + instead of whichever process happened to write last. Set/dict activity is + unioned; scalar identity (``argv``, ``python_version``, ...) is taken from the + richest shard, i.e. the one that imported the most modules: multiprocessing + workers (``python -c ...``) import a subset, the workload imports everything. + + A no-op if there are no shards (e.g. the tracer produced no report). + """ + shards: list[tuple[str, dict]] = [] + for path in sorted(glob.glob(glob.escape(base_path) + ".*")): + try: + with open(path) as handle: + shards.append((path, json.load(handle))) + except (OSError, ValueError): + continue + if not shards: + return + + # Richest shard = most imported modules -> the workload, not a worker. + primary = max((data for _, data in shards), key=lambda d: len(d.get("modules_files") or [])) + merged: dict[str, Any] = dict(primary) + + for field in _MERGE_LIST_FIELDS: + union: set[str] = set() + for _, data in shards: + union.update(data.get(field) or []) + merged[field] = sorted(union) + + for field in _MERGE_DICT_FIELDS: + combined: dict[str, Any] = {} + for _, data in shards: + for key, value in (data.get(field) or {}).items(): + # Prefer a concrete version over a None placeholder. + if key not in combined or combined[key] is None: + combined[key] = value + merged[field] = dict(sorted(combined.items())) + + with open(base_path, "w") as handle: + json.dump(merged, handle) + + for path, _ in shards: + with contextlib.suppress(OSError): + os.remove(path) + + def get_active_runtime_pythonpath(environ: Mapping[str, str]) -> tuple[str, ...]: entries: list[str] = [] for raw_path in environ.get("ROAR_RUNTIME_PYTHONPATH_ACTIVE", "").split(os.pathsep): @@ -214,8 +270,18 @@ def write_log(self) -> None: "used_packages": used_packages, "python_version": platform.python_version(), "python_implementation": platform.python_implementation(), + "pid": os.getpid(), + "ppid": os.getppid(), } - with self._real_open(self._log_file, "w") as handle: + # Write to a PER-PID shard, not the shared ROAR_LOG_FILE. Every process in + # a traced tree (litdata/DataLoader workers, HF datasets num_proc, torchrun + # ranks, any multiprocessing spawn) inherits the same ROAR_LOG_FILE and + # runs this at exit; opening it "w" means each truncates the others, so the + # surviving record was whichever process wrote LAST — often a worker with a + # subset of the imports (or none of them), not the workload. Sharding by + # pid lets merge_inject_logs() union the full tree afterwards. + shard_path = f"{self._log_file}.{os.getpid()}" + with self._real_open(shard_path, "w") as handle: json.dump(data, handle) diff --git a/roar/execution/runtime/tracer.py b/roar/execution/runtime/tracer.py index 462c1615..b95c6e17 100644 --- a/roar/execution/runtime/tracer.py +++ b/roar/execution/runtime/tracer.py @@ -21,6 +21,7 @@ from ...core.models.run import TracerResult from ...core.tracer_modes import TRACER_BACKEND_ORDER, is_valid_tracer_mode from ...execution.runtime import tracer_backends +from ...execution.runtime.inject.tracker import merge_inject_logs class TracerService: @@ -619,6 +620,13 @@ def execute( signal_handler.restore() self.logger.debug("Signal handler restored") + # Union the per-PID inject-log shards written by every process in the tree + # into the single canonical inject_log_file the collector reads. Without + # this, a multiprocessing worker's shard would be the only record (each + # process used to truncate a shared log); merging recovers the workload's + # full package/file/import set. + merge_inject_logs(inject_log_file) + end_time = time.time() duration = end_time - start_time self.logger.debug( diff --git a/tests/execution/runtime/test_inject_log_merge.py b/tests/execution/runtime/test_inject_log_merge.py new file mode 100644 index 00000000..021a0725 --- /dev/null +++ b/tests/execution/runtime/test_inject_log_merge.py @@ -0,0 +1,110 @@ +"""P0-9: in a traced process tree, every process inherits one ROAR_LOG_FILE and +used to open it "w" — so a multiprocessing worker (litdata/DataLoader/HF datasets +num_proc/torchrun) could truncate the workload's record and be the one that +survived. write_log now writes a per-PID shard and merge_inject_logs unions them, +recovering the full workload instead of whichever process wrote last. +""" + +from __future__ import annotations + +import json +import os + +from roar.execution.runtime.inject.tracker import ( + RuntimeInjectionTracker, + merge_inject_logs, +) + + +class _FakeController: + def handle_import(self, module_name, module): + return None + + +def _tracker(log_path): + return RuntimeInjectionTracker( + {"ROAR_LOG_FILE": str(log_path)}, + _FakeController(), + log_file=str(log_path), + inject_dir=str(log_path.parent / "inject"), + ) + + +def _shard(base, pid, data): + (base.parent / f"{base.name}.{pid}").write_text(json.dumps(data), encoding="utf-8") + + +def test_write_log_writes_a_per_pid_shard_not_the_shared_file(tmp_path): + log_path = tmp_path / "inject-log.json" + _tracker(log_path).write_log() + assert not log_path.exists() # the shared path is NOT truncated + assert (tmp_path / f"inject-log.json.{os.getpid()}").exists() # the shard is + + +def test_worker_shard_does_not_clobber_the_workload_record(tmp_path): + """MMA's litdata case: same command, a worker shard with argv ['-c'] and a + subset of packages, plus the workload shard with the real command and the + full set. The merge must keep the workload identity and union the packages.""" + base = tmp_path / "inject-log.json" + # A litdata worker: sparse, argv ['-c'], few modules. + _shard( + base, + 222, + { + "argv": ["-c"], + "modules_files": ["/sp/multiprocessing/spawn.py"], + "used_packages": {"litdata": "0.2.59"}, + "imported_modules": ["litdata"], + "opened_files": ["/data/shard-0.bin"], + "installed_packages": {"litdata": "0.2.59"}, + "python_version": "3.12.10", + }, + ) + # The workload: the real command, the full package set (torch/lightning/...). + _shard( + base, + 111, + { + "argv": ["train.py", "--epochs", "3"], + "modules_files": [ + "/sp/torch/__init__.py", + "/sp/lightning/__init__.py", + "/repo/train.py", + ], + "used_packages": {"torch": "2.7.0", "lightning": "2.6.5", "litdata": "0.2.59"}, + "imported_modules": ["torch", "lightning", "litdata"], + "opened_files": ["/repo/train.py"], + "installed_packages": {"torch": "2.7.0", "lightning": "2.6.5", "litdata": "0.2.59"}, + "python_version": "3.12.10", + }, + ) + + merge_inject_logs(str(base)) + merged = json.loads(base.read_text(encoding="utf-8")) + + # Identity comes from the workload (richest shard), not the ['-c'] worker. + assert merged["argv"] == ["train.py", "--epochs", "3"] + # Packages/files/imports are the UNION across the tree. + assert merged["used_packages"] == {"torch": "2.7.0", "lightning": "2.6.5", "litdata": "0.2.59"} + assert set(merged["imported_modules"]) == {"torch", "lightning", "litdata"} + assert ( + "/repo/train.py" in merged["opened_files"] and "/data/shard-0.bin" in merged["opened_files"] + ) + # Shards are consumed. + assert not list(tmp_path.glob("inject-log.json.*")) + + +def test_merge_prefers_a_concrete_version_over_none(tmp_path): + base = tmp_path / "inject-log.json" + _shard(base, 1, {"modules_files": ["/sp/a.py"], "used_packages": {"wandb": None}}) + _shard( + base, 2, {"modules_files": ["/sp/a.py", "/sp/b.py"], "used_packages": {"wandb": "0.16.0"}} + ) + merge_inject_logs(str(base)) + assert json.loads(base.read_text())["used_packages"]["wandb"] == "0.16.0" + + +def test_merge_is_noop_without_shards(tmp_path): + base = tmp_path / "inject-log.json" + merge_inject_logs(str(base)) # no shards on disk + assert not base.exists() diff --git a/tests/execution/runtime/test_runtime_tracker.py b/tests/execution/runtime/test_runtime_tracker.py index 165d2c68..a9084bc5 100644 --- a/tests/execution/runtime/test_runtime_tracker.py +++ b/tests/execution/runtime/test_runtime_tracker.py @@ -5,6 +5,7 @@ from roar.execution.runtime.inject.tracker import ( RuntimeInjectionTracker, + merge_inject_logs, ) @@ -31,6 +32,7 @@ def handle_import(self, module_name: str, module) -> None: assert tracker.patched_environ_get("VIRTUAL_ENV") == "/tmp/venv" tracker.write_log() + merge_inject_logs(str(log_path)) # write_log writes a per-PID shard; merge -> canonical payload = json.loads(log_path.read_text(encoding="utf-8")) assert str(data_path.resolve()) in payload["opened_files"] assert payload["env_reads"]["VIRTUAL_ENV"] == "/tmp/venv" @@ -75,6 +77,7 @@ def handle_import(self, module_name: str, module) -> None: sys.path.remove(str(runtime_root)) sys.modules.pop("runtime_only", None) + merge_inject_logs(str(log_path)) # write_log writes a per-PID shard; merge -> canonical payload = json.loads(log_path.read_text(encoding="utf-8")) assert str(runtime_module) not in payload["modules_files"] @@ -107,6 +110,7 @@ def handle_import(self, module_name: str, module) -> None: assert tracker.patched_environ_get("HOME") == "/home/ubuntu" tracker.write_log() + merge_inject_logs(str(log_path)) # write_log writes a per-PID shard; merge -> canonical payload = json.loads(log_path.read_text(encoding="utf-8")) env_reads = payload["env_reads"] # User-facing reads are kept; roar's reserved namespace is dropped. From af6ed6d50ca443f1bebafcd576694c71ab8e2512 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Fri, 7 Aug 2026 13:18:44 +0000 Subject: [PATCH 11/52] test: merge inject-log shards in the data-loader roundtrip (P0-9 follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_writer_reader_roundtrip_carries_python_identity calls write_log then reads the canonical inject-log path — but write_log now writes a per-PID shard, so the canonical file was empty and python_version came back ''. Merge the shard first, matching the fix already applied to test_runtime_tracker. Caught by CI across all Python versions. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/unit/test_tracer_data_loader.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_tracer_data_loader.py b/tests/unit/test_tracer_data_loader.py index 8f4e9945..b593a6be 100644 --- a/tests/unit/test_tracer_data_loader.py +++ b/tests/unit/test_tracer_data_loader.py @@ -211,7 +211,10 @@ def test_writer_reader_roundtrip_carries_python_identity(self, tmp_path: Path) - writer drops the key or the reader doesn't extract it, the loaded model's python_version is empty. """ - from roar.execution.runtime.inject.tracker import RuntimeInjectionTracker + from roar.execution.runtime.inject.tracker import ( + RuntimeInjectionTracker, + merge_inject_logs, + ) log_path = tmp_path / "inject-log.json" @@ -226,6 +229,7 @@ def handle_import(self, module_name, module) -> None: inject_dir=str(tmp_path / "inject"), ) tracker.write_log() + merge_inject_logs(str(log_path)) # write_log writes a per-PID shard; merge -> canonical data = DataLoaderService().load_python_data(str(log_path)) From 5645965c424c247ad99431541919fc938658a685 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Fri, 7 Aug 2026 12:34:41 +0000 Subject: [PATCH 12/52] sitecustomize: don't let roar's env shadow the workload's sys.path (P0-14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit roar makes itself importable in a traced child by putting entries on ROAR_RUNTIME_PYTHONPATH, which sitecustomize applied by prepending ALL of them. On a cross-interpreter run (roar under system 3.10, workload venv 3.12) that put roar's host dist-packages at sys.path[0], shadowing the recorded pins — the run executed against host packages. It crashed loudly only because the host's pyparsing was 5 years stale; a merely-different host package would import fine and certify GREEN for the wrong reason (and the venv-vs-manifest guard can't see it — the venv is right; the child's sys.path is wrong). Fix: split ROAR_RUNTIME_PYTHONPATH by precedence. - roar's ABI-matched runtime CACHE (~/.cache/roar/runtime//…) stays PREPENDED — it must beat the child's wrong-ABI/stale system copies (the original typing_extensions-4.15-vs-system-4.4 fix). - everything else (roar's host site-packages) is now APPENDED, so the workload's own venv always wins. roar's core injection is pure-Python, so it's still importable via the appended path; ABI-specific backend deps are handled separately by the existing runtime gate. Cache-root detection is inlined (matching lazy_install.runtime_cache_root) because this runs before roar is importable. Tests: cache entry prepended (must-win preserved); host entry appended, not at front (P0-14); early-return unchanged. Plus an integration test running a roar-less child that confirms a workload package beats a host one (non-vacuous: fails without the fix) and the cache still beats the workload. Harness note: installing roar under the recorded interpreter (`uv tool install --python `) sidesteps this by making roar's interpreter == the child's; this fix removes the silent-false-pass regardless. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../execution/runtime/inject/sitecustomize.py | 57 ++++++--- .../runtime/test_sitecustomize_path_order.py | 115 ++++++++++-------- .../test_no_crossenv_syspath_shadow.py | 84 +++++++++++++ 3 files changed, 190 insertions(+), 66 deletions(-) create mode 100644 tests/integration/test_no_crossenv_syspath_shadow.py diff --git a/roar/execution/runtime/inject/sitecustomize.py b/roar/execution/runtime/inject/sitecustomize.py index 3e44a03f..011f8725 100644 --- a/roar/execution/runtime/inject/sitecustomize.py +++ b/roar/execution/runtime/inject/sitecustomize.py @@ -5,19 +5,40 @@ import sys -def _prepend_roar_runtime_pythonpath() -> None: - """Prepend ``ROAR_RUNTIME_PYTHONPATH`` entries to ``sys.path`` (in order). - - When the traced Python has a lazy-installed ABI-matched runtime tree on - ``ROAR_RUNTIME_PYTHONPATH``, that tree must beat system site-packages — - the system copies are the wrong-ABI ones, which is exactly why we - installed the tree in the first place. Prepending the whole list in - declared order (cache, then bundled fallbacks) keeps the lazy-install - cache at ``sys.path[0]``. - - Logic is inlined (rather than imported from elsewhere in roar) because - this runs *before* roar is necessarily importable — making roar - importable is exactly what this function does. +def _roar_runtime_cache_root() -> str: + """``$XDG_CACHE_HOME/roar/runtime`` (default ``~/.cache/roar/runtime``). + + Inlined to match ``lazy_install.runtime_cache_root()`` — this runs before + roar is importable, so it can't call into roar. + """ + xdg = os.environ.get("XDG_CACHE_HOME") + base = xdg if xdg else os.path.join(os.path.expanduser("~"), ".cache") + return os.path.abspath(os.path.join(base, "roar", "runtime")) + + +def _add_roar_runtime_pythonpath() -> None: + """Make roar importable in the traced process **without letting roar's own + environment shadow the workload's recorded packages**. + + ``ROAR_RUNTIME_PYTHONPATH`` carries two very different kinds of entry: + + - roar's lazy-installed **ABI-matched runtime cache** + (``~/.cache/roar/runtime//site-packages``). This *must* beat the + system's wrong-ABI copies — that is why it was installed — so it is + **prepended**. + - roar's package root / the parent interpreter's **site-packages**, added so + a non-editable or cross-interpreter child can import roar at all. These are + **appended**, so the workload's own venv always wins. + + Prepending the second kind was **P0-14**: when roar ran under a different + interpreter than the child (e.g. roar under system 3.10, workload venv 3.12), + its host ``dist-packages`` landed at ``sys.path[0]`` and shadowed the recorded + pins — the run executed against host packages and could certify GREEN for the + wrong reason. roar's core injection is pure-Python, so appending still leaves + it importable; ABI-specific backend deps are handled separately by the runtime + gate below. + + Inlined (not imported from roar) because this runs before roar is importable. """ if importlib.util.find_spec("roar") is not None: return @@ -28,11 +49,17 @@ def _prepend_roar_runtime_pythonpath() -> None: ] if not new_paths: return - sys.path[:0] = new_paths + cache_root = _roar_runtime_cache_root() + must_win = [p for p in new_paths if os.path.abspath(p).startswith(cache_root + os.sep)] + others = [p for p in new_paths if p not in must_win] + if must_win: + sys.path[:0] = must_win # ABI-matched cache must beat wrong-ABI system copies + if others: + sys.path.extend(others) # roar's env must NOT shadow the workload's venv (P0-14) os.environ["ROAR_RUNTIME_PYTHONPATH_ACTIVE"] = os.pathsep.join(new_paths) -_prepend_roar_runtime_pythonpath() +_add_roar_runtime_pythonpath() from roar.execution.framework.runtime_imports import RuntimeImportController from roar.execution.runtime.inject.support import ( diff --git a/tests/execution/runtime/test_sitecustomize_path_order.py b/tests/execution/runtime/test_sitecustomize_path_order.py index 11830572..c2e71e4d 100644 --- a/tests/execution/runtime/test_sitecustomize_path_order.py +++ b/tests/execution/runtime/test_sitecustomize_path_order.py @@ -1,15 +1,19 @@ -"""sitecustomize prepends ROAR_RUNTIME_PYTHONPATH entries (in order). - -Behavior under test: when the traced Python doesn't already have roar -importable (the cross-Python lazy-install scenario), ``sitecustomize.py`` -must put the entries from ``ROAR_RUNTIME_PYTHONPATH`` at the *front* of -``sys.path``, preserving the declared order. Appending (the old behavior) -lets the system's stale site-packages win — which is the friction-journal -bug where lazy-installed ``typing_extensions`` 4.15.0 lost to the -system's 4.4.x. - -Tested via subprocess so we exercise the real sitecustomize module-import -side effects without polluting the test process's ``sys.path``. +"""sitecustomize places ROAR_RUNTIME_PYTHONPATH entries with the right precedence. + +When the traced Python can't already import roar (the cross-Python / +lazy-install scenario), ``sitecustomize.py`` adds ``ROAR_RUNTIME_PYTHONPATH`` +entries to ``sys.path`` with two different precedences: + +- roar's **ABI-matched runtime cache** (``~/.cache/roar/runtime//…``) is + **prepended** — it must beat the child's system copies (the friction-journal + bug: a lazy-installed ``typing_extensions`` 4.15 losing to system 4.4). +- everything else (roar's host site-packages, added only so a cross-interpreter + child can import roar) is **appended** — prepending it was P0-14: roar's host + packages shadowed the workload's recorded pins and the run executed against + host packages. + +Tested via subprocess so we exercise the real sitecustomize module-import side +effects without polluting the test process's ``sys.path``. """ from __future__ import annotations @@ -22,11 +26,21 @@ SOURCE_ROOT = Path(__file__).resolve().parents[3] +# Patches find_spec("roar") -> None so the add-path codepath runs, then imports +# sitecustomize. Callers append their own print statements. +_PATCH_AND_IMPORT = """ +import importlib.util, importlib, sys +_real = importlib.util.find_spec +importlib.util.find_spec = lambda name, *a, **k: None if name == "roar" else _real(name, *a, **k) +importlib.import_module("roar.execution.runtime.inject.sitecustomize") +""" + def _run_python( code: str, *, roar_runtime_pythonpath: str | None = None, + extra_env: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: """Run a subprocess Python with sitecustomize loaded from this source tree.""" env = dict(os.environ) @@ -38,6 +52,8 @@ def _run_python( else: env.pop("ROAR_RUNTIME_PYTHONPATH", None) env.pop("ROAR_WRAP", None) # skip the backend-dispatch gate; we only care about path order + if extra_env: + env.update(extra_env) return subprocess.run( [sys.executable, "-c", code], capture_output=True, @@ -49,61 +65,58 @@ def _run_python( ) -def test_runtime_pythonpath_entries_land_at_front_in_declared_order(tmp_path: Path) -> None: - """When roar isn't already importable, ROAR_RUNTIME_PYTHONPATH wins.""" - fake_runtime = tmp_path / "fake-runtime" - fake_runtime.mkdir() - fake_other = tmp_path / "fake-other" - fake_other.mkdir() +def test_abi_matched_cache_is_prepended(tmp_path: Path) -> None: + """roar's ABI-matched runtime cache must beat system copies -> prepended.""" + cache_home = tmp_path / "xdg" + cache_dir = cache_home / "roar" / "runtime" / "cp999" / "site-packages" + cache_dir.mkdir(parents=True) - # Force find_spec("roar") to return None by monkey-patching it before - # sitecustomize runs. We mark roar's site-packages location empty for the - # purposes of this subprocess by inserting a stub finder ahead of it that - # claims "roar is missing" — that's what triggers the prepend codepath. - code = textwrap.dedent( - """ - import importlib.util - import sys + code = _PATCH_AND_IMPORT + "print(f'first={sys.path[0]}')\n" + result = _run_python( + code, + roar_runtime_pythonpath=str(cache_dir), + extra_env={"XDG_CACHE_HOME": str(cache_home)}, + ) + assert result.returncode == 0, result.stderr + assert f"first={cache_dir}" in result.stdout, result.stdout - _real_find_spec = importlib.util.find_spec - def _patched_find_spec(name, *args, **kwargs): - if name == "roar": - return None - return _real_find_spec(name, *args, **kwargs) - importlib.util.find_spec = _patched_find_spec - import importlib - importlib.import_module("roar.execution.runtime.inject.sitecustomize") - # The prepend has run; assert the entries are at the front, in order. - print(f"first={sys.path[0]}") - print(f"second={sys.path[1]}") +def test_host_site_packages_are_appended_not_prepended(tmp_path: Path) -> None: + """P0-14: a non-cache runtime entry is appended, so it can't shadow the + workload — present on sys.path, but not at the front.""" + fake_host = tmp_path / "fake-host" + fake_host.mkdir() + + code = _PATCH_AND_IMPORT + textwrap.dedent( + f""" + target = {str(fake_host)!r} + print("present=" + str(target in sys.path)) + print("at_front=" + str(sys.path[0] == target)) + print("at_back=" + str(sys.path[-1] == target)) """ ) - result = _run_python( - code, - roar_runtime_pythonpath=os.pathsep.join([str(fake_runtime), str(fake_other)]), - ) + result = _run_python(code, roar_runtime_pythonpath=str(fake_host)) assert result.returncode == 0, result.stderr out = result.stdout - assert f"first={fake_runtime}" in out, out - assert f"second={fake_other}" in out, out + assert "present=True" in out, out + assert "at_front=False" in out, out + assert "at_back=True" in out, out -def test_no_prepend_when_roar_already_importable(tmp_path: Path) -> None: - """When roar is already importable, the function early-returns and leaves sys.path alone.""" +def test_no_change_when_roar_already_importable(tmp_path: Path) -> None: + """When roar is already importable, the function early-returns and leaves + sys.path alone.""" fake_runtime = tmp_path / "fake-runtime" fake_runtime.mkdir() code = textwrap.dedent( f""" - import importlib - import sys - # Roar IS importable (PYTHONPATH points at source root). Prepend should no-op. + import importlib, sys + # Roar IS importable (PYTHONPATH points at source root). Add-path should no-op. importlib.import_module("roar.execution.runtime.inject.sitecustomize") target = {str(fake_runtime)!r} - in_top_three = target in sys.path[:3] - print("in_top_three=" + str(in_top_three)) + print("present=" + str(target in sys.path)) """ ) result = _run_python(code, roar_runtime_pythonpath=str(fake_runtime)) assert result.returncode == 0, result.stderr - assert "in_top_three=False" in result.stdout, result.stdout + assert "present=False" in result.stdout, result.stdout diff --git a/tests/integration/test_no_crossenv_syspath_shadow.py b/tests/integration/test_no_crossenv_syspath_shadow.py new file mode 100644 index 00000000..1cf10398 --- /dev/null +++ b/tests/integration/test_no_crossenv_syspath_shadow.py @@ -0,0 +1,84 @@ +"""P0-14: roar's runtime injection must not let roar's own environment shadow the +workload's recorded packages on ``sys.path``. + +roar makes itself importable in a traced child by putting entries on +``ROAR_RUNTIME_PYTHONPATH``; ``sitecustomize`` applies them. Previously *all* of +them were prepended, so on a cross-interpreter run roar's host ``dist-packages`` +landed at ``sys.path[0]`` and shadowed the recorded pins (the run executed +against host packages, and could certify GREEN for the wrong reason). The fix: +prepend only roar's ABI-matched runtime **cache**; append everything else so the +workload's venv wins. + +These run a roar-less child interpreter with a ``sitecustomize`` on +``PYTHONPATH`` and assert which copy of a shadowed module wins. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import venv +from pathlib import Path + +INJECT_DIR = Path(__file__).resolve().parents[2] / "roar" / "execution" / "runtime" / "inject" + + +def _roarless_python(tmp_path: Path) -> Path: + """A Python that cannot already import roar (so the injection path runs).""" + child = tmp_path / "child" + venv.EnvBuilder(with_pip=False).create(str(child)) + return child / ("Scripts" if sys.platform == "win32" else "bin") / "python" + + +def _write_pkg(root: Path, mark: str) -> None: + root.mkdir(parents=True, exist_ok=True) + (root / "shadowpkg.py").write_text(f"MARK = {mark!r}\n", encoding="utf-8") + + +def _run(child_py: Path, env: dict) -> str: + r = subprocess.run( + [str(child_py), "-c", "import shadowpkg; print('WINNER=' + shadowpkg.MARK)"], + env=env, + capture_output=True, + text=True, + ) + for line in r.stdout.splitlines(): + if line.startswith("WINNER="): + return line[len("WINNER=") :] + raise AssertionError(f"no winner line.\nstdout={r.stdout!r}\nstderr={r.stderr!r}") + + +def test_host_site_packages_do_not_shadow_the_workload(tmp_path): + """A non-cache runtime entry (roar's host site-packages) is APPENDED, so the + workload's own copy (on PYTHONPATH) wins. Before the fix it was prepended and + 'host' won — a silent execution against host packages.""" + child_py = _roarless_python(tmp_path) + _write_pkg(tmp_path / "workload", "workload") + _write_pkg(tmp_path / "host", "host") + + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join([str(INJECT_DIR), str(tmp_path / "workload")]) + env["ROAR_RUNTIME_PYTHONPATH"] = str(tmp_path / "host") + env.pop("ROAR_WRAP", None) + + assert _run(child_py, env) == "workload" + + +def test_abi_matched_cache_still_beats_the_workload(tmp_path): + """roar's ABI-matched runtime cache (~/.cache/roar/runtime//...) is still + PREPENDED — it must beat the child's wrong-ABI system copies. Here it wins over + the workload copy, confirming the must-win branch is preserved.""" + child_py = _roarless_python(tmp_path) + cache_home = tmp_path / "xdg" + cache_pkg = cache_home / "roar" / "runtime" / "cp999" / "site-packages" + _write_pkg(cache_pkg, "cache") + _write_pkg(tmp_path / "workload", "workload") + + env = dict(os.environ) + env["XDG_CACHE_HOME"] = str(cache_home) + env["PYTHONPATH"] = os.pathsep.join([str(INJECT_DIR), str(tmp_path / "workload")]) + env["ROAR_RUNTIME_PYTHONPATH"] = str(cache_pkg) + env.pop("ROAR_WRAP", None) + + assert _run(child_py, env) == "cache" From cf8f36d4ccfdf39d1aebaea87d32ff9486cb6afd Mon Sep 17 00:00:00 2001 From: Trevor Basinger Date: Fri, 7 Aug 2026 14:39:54 +0000 Subject: [PATCH 13/52] fix(runtime): preserve workload package precedence --- .../execution/runtime/inject/sitecustomize.py | 142 +++++++++++++++++- roar/execution/runtime/inject/tracker.py | 27 +++- roar/execution/runtime/lazy_install.py | 6 +- roar/execution/runtime/tracer.py | 5 +- .../execution/runtime/test_runtime_tracker.py | 38 +++++ .../runtime/test_sitecustomize_path_order.py | 81 +++++++++- .../test_cross_python_runtime_repair.py | 8 +- .../test_no_crossenv_syspath_shadow.py | 121 +++++++++++++-- 8 files changed, 392 insertions(+), 36 deletions(-) diff --git a/roar/execution/runtime/inject/sitecustomize.py b/roar/execution/runtime/inject/sitecustomize.py index 011f8725..3e25ca64 100644 --- a/roar/execution/runtime/inject/sitecustomize.py +++ b/roar/execution/runtime/inject/sitecustomize.py @@ -1,9 +1,12 @@ # ruff: noqa: E402 import atexit +import importlib.machinery import importlib.util import os import sys +_RUNTIME_CACHE_COLLISIONS_ENV = "ROAR_RUNTIME_CACHE_COLLISIONS" + def _roar_runtime_cache_root() -> str: """``$XDG_CACHE_HOME/roar/runtime`` (default ``~/.cache/roar/runtime``). @@ -16,6 +19,116 @@ def _roar_runtime_cache_root() -> str: return os.path.abspath(os.path.join(base, "roar", "runtime")) +def _path_key(path: str) -> str: + """Normalize a path for comparisons without changing import ordering.""" + return os.path.normcase(os.path.realpath(os.path.abspath(path or os.curdir))) + + +def _runtime_pythonpath_entries() -> list[str]: + return [ + path for path in os.environ.get("ROAR_RUNTIME_PYTHONPATH", "").split(os.pathsep) if path + ] + + +def _workload_search_path() -> list[str]: + """Return the original workload import roots, excluding Roar-owned paths.""" + roar_paths = {_path_key(path) for path in _runtime_pythonpath_entries()} + roar_paths.update( + _path_key(path) + for path in os.environ.get("ROAR_RUNTIME_PYTHONPATH_ACTIVE", "").split(os.pathsep) + if path + ) + roar_paths.add(_path_key(os.path.dirname(os.path.abspath(__file__)))) + return [path for path in sys.path if _path_key(path) not in roar_paths] + + +def _top_level_import_names(paths: list[str]) -> set[str]: + """Discover import names supplied by one or more site-packages trees.""" + names: set[str] = set() + import_suffixes = sorted( + { + *importlib.machinery.SOURCE_SUFFIXES, + *importlib.machinery.BYTECODE_SUFFIXES, + *importlib.machinery.EXTENSION_SUFFIXES, + }, + key=len, + reverse=True, + ) + for path in paths: + try: + entries = os.scandir(path) + except OSError: + continue + with entries: + for entry in entries: + entry_name = entry.name + if entry_name.startswith(".") or entry_name == "__pycache__": + continue + if entry_name.endswith((".dist-info", ".egg-info", ".data")): + continue + try: + if entry.is_dir(): + candidate = entry_name + elif entry.is_file(): + candidate = "" + for suffix in import_suffixes: + if entry_name.endswith(suffix): + candidate = entry_name[: -len(suffix)] + break + else: + continue + except OSError: + continue + if candidate.isidentifier(): + names.add(candidate) + return names + + +def _runtime_cache_collisions(cache_paths: list[str]) -> tuple[str, ...]: + """Names a cache would shadow on the workload's unmodified search path.""" + workload_paths = _workload_search_path() + collisions: list[str] = [] + for name in sorted(_top_level_import_names(cache_paths)): + try: + spec = importlib.machinery.PathFinder.find_spec(name, workload_paths) + except Exception: + # Detection uncertainty must degrade rather than risk changing the workload. + collisions.append(name) + continue + if spec is not None: + collisions.append(name) + return tuple(collisions) + + +def _record_runtime_cache_collisions(collisions: tuple[str, ...]) -> None: + if collisions: + os.environ[_RUNTIME_CACHE_COLLISIONS_ENV] = ",".join(collisions) + else: + os.environ.pop(_RUNTIME_CACHE_COLLISIONS_ENV, None) + + +def _set_active_runtime_paths(paths: list[str]) -> None: + if paths: + os.environ["ROAR_RUNTIME_PYTHONPATH_ACTIVE"] = os.pathsep.join(paths) + else: + os.environ.pop("ROAR_RUNTIME_PYTHONPATH_ACTIVE", None) + + +def _add_active_runtime_path(path: str, *, prepend: bool = False) -> None: + active = [ + entry + for entry in os.environ.get("ROAR_RUNTIME_PYTHONPATH_ACTIVE", "").split(os.pathsep) + if entry + ] + if path in active: + return + if prepend: + active.insert(0, path) + else: + active.append(path) + _set_active_runtime_paths(active) + + def _add_roar_runtime_pythonpath() -> None: """Make roar importable in the traced process **without letting roar's own environment shadow the workload's recorded packages**. @@ -25,7 +138,7 @@ def _add_roar_runtime_pythonpath() -> None: - roar's lazy-installed **ABI-matched runtime cache** (``~/.cache/roar/runtime//site-packages``). This *must* beat the system's wrong-ABI copies — that is why it was installed — so it is - **prepended**. + **prepended only when none of its import names overlap the workload**. - roar's package root / the parent interpreter's **site-packages**, added so a non-editable or cross-interpreter child can import roar at all. These are **appended**, so the workload's own venv always wins. @@ -42,21 +155,22 @@ def _add_roar_runtime_pythonpath() -> None: """ if importlib.util.find_spec("roar") is not None: return - new_paths = [ - path - for path in os.environ.get("ROAR_RUNTIME_PYTHONPATH", "").split(os.pathsep) - if path and path not in sys.path - ] + new_paths = [path for path in _runtime_pythonpath_entries() if path not in sys.path] if not new_paths: return cache_root = _roar_runtime_cache_root() must_win = [p for p in new_paths if os.path.abspath(p).startswith(cache_root + os.sep)] others = [p for p in new_paths if p not in must_win] - if must_win: + collisions = _runtime_cache_collisions(must_win) if must_win else () + _record_runtime_cache_collisions(collisions) + active_paths: list[str] = [] + if must_win and not collisions: sys.path[:0] = must_win # ABI-matched cache must beat wrong-ABI system copies + active_paths.extend(must_win) if others: sys.path.extend(others) # roar's env must NOT shadow the workload's venv (P0-14) - os.environ["ROAR_RUNTIME_PYTHONPATH_ACTIVE"] = os.pathsep.join(new_paths) + active_paths.extend(others) + _set_active_runtime_paths(active_paths) _add_roar_runtime_pythonpath() @@ -121,14 +235,26 @@ def _repair_runtime_in_process(expected_soabi: str) -> bool: if tree is None: return False tree_str = str(tree) + collisions = _runtime_cache_collisions([tree_str]) + _record_runtime_cache_collisions(collisions) + if collisions: + return False if tree_str not in sys.path: sys.path.insert(0, tree_str) + _add_active_runtime_path(tree_str, prepend=True) return matching_compiled_pydantic_core(sys.path, expected_soabi) def _runtime_gate_degrade_message(running_abi: tuple[int, int]) -> str: + collisions = os.environ.get(_RUNTIME_CACHE_COLLISIONS_ENV, "") + collision_message = ( + f" Runtime cache disabled to preserve workload imports: {collisions}.\n" + if collisions + else "" + ) return ( f"roar: no ABI-matched runtime found for Python {running_abi[0]}.{running_abi[1]}.\n" + f"{collision_message}" f" Backend integrations (Ray, OSMO) are disabled for this run.\n" f" File I/O is still captured.\n" f" Fix one of:\n" diff --git a/roar/execution/runtime/inject/tracker.py b/roar/execution/runtime/inject/tracker.py index 24d067df..0511b53e 100644 --- a/roar/execution/runtime/inject/tracker.py +++ b/roar/execution/runtime/inject/tracker.py @@ -48,17 +48,27 @@ def get_loaded_shared_libs(real_open) -> list[str]: return sorted(libs) -def get_installed_packages() -> dict[str, str]: +def get_installed_packages( + excluded_paths: Sequence[str] = (), +) -> dict[str, str]: packages: dict[str, str] = {} try: from importlib import metadata as importlib_metadata for dist in importlib_metadata.distributions(): + try: + distribution_root = str(dist.locate_file("")) + except Exception: + distribution_root = "" + if distribution_root and is_under_any_runtime_path(distribution_root, excluded_paths): + continue metadata = cast(Mapping[str, str], dist.metadata) name = metadata.get("Name", None) version = metadata.get("Version", None) if name and version: - packages[name] = version + # Import resolution and distributions() both follow sys.path order. + # Preserve the first visible workload distribution for duplicate names. + packages.setdefault(name, version) except Exception: pass return packages @@ -119,8 +129,15 @@ def get_active_runtime_pythonpath(environ: Mapping[str, str]) -> tuple[str, ...] def is_under_any_runtime_path(path: str, runtime_paths: Sequence[str]) -> bool: if not runtime_paths: return False - abs_path = os.path.abspath(path) - return any(abs_path.startswith(runtime_path) for runtime_path in runtime_paths) + abs_path = os.path.normcase(os.path.abspath(path)) + for runtime_path in runtime_paths: + abs_runtime_path = os.path.normcase(os.path.abspath(runtime_path)) + try: + if os.path.commonpath([abs_path, abs_runtime_path]) == abs_runtime_path: + return True + except ValueError: + continue + return False class RuntimeInjectionTracker: @@ -197,7 +214,7 @@ def write_log(self) -> None: runtime_pythonpath, ) ) - installed_packages = get_installed_packages() + installed_packages = get_installed_packages(excluded_paths=runtime_pythonpath) used_packages = get_used_packages(modules_files, installed_packages) data = { "opened_files": sorted(self.opened_files), diff --git a/roar/execution/runtime/lazy_install.py b/roar/execution/runtime/lazy_install.py index b5afc2f0..66a5ee05 100644 --- a/roar/execution/runtime/lazy_install.py +++ b/roar/execution/runtime/lazy_install.py @@ -6,9 +6,9 @@ module installs a matching tree of runtime deps on demand into a per-ABI cache directory under ``~/.cache/roar/runtime//``. -``sitecustomize.py``'s ``_append_roar_runtime_pythonpath`` prepends the -cache directory to ``sys.path`` in the traced process, so imports there -resolve to the ABI-matched copies before reaching roar's bundled tree. +``sitecustomize.py`` prepends the cache directory only when its import names +do not collide with the workload. A collision degrades optional backend +dispatch instead of changing which packages the workload imports. """ from __future__ import annotations diff --git a/roar/execution/runtime/tracer.py b/roar/execution/runtime/tracer.py index 462c1615..074f6025 100644 --- a/roar/execution/runtime/tracer.py +++ b/roar/execution/runtime/tracer.py @@ -114,8 +114,9 @@ def _lazy_install_runtime_entries( ) -> list[str]: """Probe the target Python and lazy-install a matching runtime tree on mismatch. - Returns a list of site-packages paths to prepend to - ``ROAR_RUNTIME_PYTHONPATH``. Empty on: + Returns a list of ABI-matched site-packages paths for + ``ROAR_RUNTIME_PYTHONPATH``. ``sitecustomize`` activates a returned + path only when it cannot shadow a workload import. Empty on: - non-Python targets (bash, make, etc.) — can't probe a python ABI; - matching ABI — bundled deps work as-is; - ``runtime.install = skip`` — opted out; diff --git a/tests/execution/runtime/test_runtime_tracker.py b/tests/execution/runtime/test_runtime_tracker.py index 165d2c68..3381f440 100644 --- a/tests/execution/runtime/test_runtime_tracker.py +++ b/tests/execution/runtime/test_runtime_tracker.py @@ -2,9 +2,15 @@ import json import sys +from pathlib import Path + +import pytest from roar.execution.runtime.inject.tracker import ( RuntimeInjectionTracker, + get_active_runtime_pythonpath, + get_installed_packages, + get_used_packages, ) @@ -79,6 +85,38 @@ def handle_import(self, module_name: str, module) -> None: assert str(runtime_module) not in payload["modules_files"] +def test_package_version_comes_from_imported_workload_distribution( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + workload = tmp_path / "workload" / "site-packages" + runtime = tmp_path / "runtime" / "site-packages" + + def write_distribution(root: Path, version: str) -> Path: + root.mkdir(parents=True) + module = root / "shadowpkg.py" + module.write_text("VALUE = 1\n", encoding="utf-8") + metadata = root / f"shadowpkg-{version}.dist-info" + metadata.mkdir() + (metadata / "METADATA").write_text( + f"Metadata-Version: 2.1\nName: shadowpkg\nVersion: {version}\n", + encoding="utf-8", + ) + (metadata / "top_level.txt").write_text("shadowpkg\n", encoding="utf-8") + return module + + workload_module = write_distribution(workload, "1.0") + write_distribution(runtime, "9.0") + monkeypatch.setattr(sys, "path", [str(workload), str(runtime), *sys.path]) + runtime_paths = get_active_runtime_pythonpath({"ROAR_RUNTIME_PYTHONPATH_ACTIVE": str(runtime)}) + + installed = get_installed_packages(excluded_paths=runtime_paths) + used = get_used_packages([str(workload_module)], installed) + + assert installed["shadowpkg"] == "1.0" + assert used["shadowpkg"] == "1.0" + + def test_runtime_tracker_excludes_roar_internal_env_reads(tmp_path) -> None: """roar's own injected vars must not leak into captured env_reads (issue #164).""" log_path = tmp_path / "inject-log.json" diff --git a/tests/execution/runtime/test_sitecustomize_path_order.py b/tests/execution/runtime/test_sitecustomize_path_order.py index c2e71e4d..96f08b5f 100644 --- a/tests/execution/runtime/test_sitecustomize_path_order.py +++ b/tests/execution/runtime/test_sitecustomize_path_order.py @@ -1,12 +1,12 @@ """sitecustomize places ROAR_RUNTIME_PYTHONPATH entries with the right precedence. When the traced Python can't already import roar (the cross-Python / -lazy-install scenario), ``sitecustomize.py`` adds ``ROAR_RUNTIME_PYTHONPATH`` -entries to ``sys.path`` with two different precedences: +lazy-install scenario), ``sitecustomize.py`` adds safe +``ROAR_RUNTIME_PYTHONPATH`` entries to ``sys.path`` with two precedences: -- roar's **ABI-matched runtime cache** (``~/.cache/roar/runtime//…``) is - **prepended** — it must beat the child's system copies (the friction-journal - bug: a lazy-installed ``typing_extensions`` 4.15 losing to system 4.4). +- a non-conflicting **ABI-matched runtime cache** is **prepended** so it can + beat the child's system copies; +- a cache with any workload import-name collision is not activated; - everything else (roar's host site-packages, added only so a cross-interpreter child can import roar) is **appended** — prepending it was P0-14: roar's host packages shadowed the workload's recorded pins and the run executed against @@ -32,7 +32,7 @@ import importlib.util, importlib, sys _real = importlib.util.find_spec importlib.util.find_spec = lambda name, *a, **k: None if name == "roar" else _real(name, *a, **k) -importlib.import_module("roar.execution.runtime.inject.sitecustomize") +_sitecustomize = importlib.import_module("roar.execution.runtime.inject.sitecustomize") """ @@ -81,6 +81,75 @@ def test_abi_matched_cache_is_prepended(tmp_path: Path) -> None: assert f"first={cache_dir}" in result.stdout, result.stdout +def test_abi_cache_with_workload_collision_is_not_activated(tmp_path: Path) -> None: + cache_home = tmp_path / "xdg" + cache_dir = cache_home / "roar" / "runtime" / "cp999" / "site-packages" + workload_dir = tmp_path / "workload" + cache_dir.mkdir(parents=True) + workload_dir.mkdir() + (cache_dir / "shadowpkg.py").write_text("MARK = 'cache'\n", encoding="utf-8") + (workload_dir / "shadowpkg.py").write_text("MARK = 'workload'\n", encoding="utf-8") + + code = _PATCH_AND_IMPORT + textwrap.dedent( + f""" + import os, shadowpkg + target = {str(cache_dir)!r} + print("cache_present=" + str(target in sys.path)) + print("winner=" + shadowpkg.MARK) + print("collisions=" + os.environ.get("ROAR_RUNTIME_CACHE_COLLISIONS", "")) + """ + ) + result = _run_python( + code, + roar_runtime_pythonpath=str(cache_dir), + extra_env={ + "XDG_CACHE_HOME": str(cache_home), + "PYTHONPATH": os.pathsep.join([str(SOURCE_ROOT), str(workload_dir)]), + }, + ) + assert result.returncode == 0, result.stderr + assert "cache_present=False" in result.stdout, result.stdout + assert "winner=workload" in result.stdout, result.stdout + assert "collisions=shadowpkg" in result.stdout, result.stdout + + +def test_in_process_repair_degrades_instead_of_activating_a_colliding_cache( + tmp_path: Path, +) -> None: + cache_dir = tmp_path / "cache" / "site-packages" + workload_dir = tmp_path / "workload" + cache_dir.mkdir(parents=True) + workload_dir.mkdir() + (cache_dir / "shadowpkg.py").write_text("MARK = 'cache'\n", encoding="utf-8") + (workload_dir / "shadowpkg.py").write_text("MARK = 'workload'\n", encoding="utf-8") + + code = _PATCH_AND_IMPORT + textwrap.dedent( + f""" + import os + from pathlib import Path + from roar.execution.runtime import lazy_install + + lazy_install.ensure_runtime = lambda **_kwargs: Path({str(cache_dir)!r}) + repaired = _sitecustomize._repair_runtime_in_process("cpython-999") + print("repaired=" + str(repaired)) + print("cache_present=" + str({str(cache_dir)!r} in sys.path)) + print("collisions=" + os.environ.get("ROAR_RUNTIME_CACHE_COLLISIONS", "")) + print(_sitecustomize._runtime_gate_degrade_message((3, 99))) + """ + ) + result = _run_python( + code, + extra_env={ + "PYTHONPATH": os.pathsep.join([str(SOURCE_ROOT), str(workload_dir)]), + }, + ) + assert result.returncode == 0, result.stderr + assert "repaired=False" in result.stdout, result.stdout + assert "cache_present=False" in result.stdout, result.stdout + assert "collisions=shadowpkg" in result.stdout, result.stdout + assert "Runtime cache disabled to preserve workload imports: shadowpkg." in result.stdout + + def test_host_site_packages_are_appended_not_prepended(tmp_path: Path) -> None: """P0-14: a non-cache runtime entry is appended, so it can't shadow the workload — present on sys.path, but not at the front.""" diff --git a/tests/integration/test_cross_python_runtime_repair.py b/tests/integration/test_cross_python_runtime_repair.py index addce649..cac4da8e 100644 --- a/tests/integration/test_cross_python_runtime_repair.py +++ b/tests/integration/test_cross_python_runtime_repair.py @@ -32,8 +32,9 @@ _INJECT_DIR = str(Path(roar.__file__).resolve().parent / "execution" / "runtime" / "inject") # Roar's importable root + the site-packages carrying the *current* (and so, -# for a different-ABI worker, wrong-ABI) pydantic_core — mirrors what the tracer -# puts on ROAR_RUNTIME_PYTHONPATH for a cross-Python child. +# for a different-ABI worker, wrong-ABI) pydantic_core. The tracer owns these +# roots and declares them on ROAR_RUNTIME_PYTHONPATH; the workload's PYTHONPATH +# contains only the startup hook. _SOURCE_ROOT = str(Path(roar.__file__).resolve().parent.parent) _CURRENT_SITE_PACKAGES = str(Path(pydantic_core.__file__).resolve().parent.parent) @@ -103,7 +104,8 @@ def test_wrapper_launch_repairs_runtime_in_process_and_installs_once(tmp_path: P env = { **os.environ, - "PYTHONPATH": os.pathsep.join([_INJECT_DIR, _SOURCE_ROOT, _CURRENT_SITE_PACKAGES]), + "PYTHONPATH": _INJECT_DIR, + "ROAR_RUNTIME_PYTHONPATH": os.pathsep.join([_SOURCE_ROOT, _CURRENT_SITE_PACKAGES]), "ROAR_WRAP": "1", "XDG_CACHE_HOME": str(tmp_path / "xdg"), "PAYLOAD": _WORKER_PAYLOAD, diff --git a/tests/integration/test_no_crossenv_syspath_shadow.py b/tests/integration/test_no_crossenv_syspath_shadow.py index 1cf10398..2f1352bb 100644 --- a/tests/integration/test_no_crossenv_syspath_shadow.py +++ b/tests/integration/test_no_crossenv_syspath_shadow.py @@ -1,13 +1,13 @@ """P0-14: roar's runtime injection must not let roar's own environment shadow the -workload's recorded packages on ``sys.path``. +workload's recorded packages on ``sys.path`` or in captured provenance. roar makes itself importable in a traced child by putting entries on ``ROAR_RUNTIME_PYTHONPATH``; ``sitecustomize`` applies them. Previously *all* of them were prepended, so on a cross-interpreter run roar's host ``dist-packages`` landed at ``sys.path[0]`` and shadowed the recorded pins (the run executed against host packages, and could certify GREEN for the wrong reason). The fix: -prepend only roar's ABI-matched runtime **cache**; append everything else so the -workload's venv wins. +append roar's host environment, and activate the ABI-matched runtime **cache** +only when none of its import names collide with the workload. These run a roar-less child interpreter with a ``sitecustomize`` on ``PYTHONPATH`` and assert which copy of a shadowed module wins. @@ -15,13 +15,22 @@ from __future__ import annotations +import json import os +import platform +import sqlite3 import subprocess import sys import venv +from collections.abc import Callable from pathlib import Path +import pytest + +import tests.conftest as test_conftest + INJECT_DIR = Path(__file__).resolve().parents[2] / "roar" / "execution" / "runtime" / "inject" +SOURCE_ROOT = Path(__file__).resolve().parents[2] def _roarless_python(tmp_path: Path) -> Path: @@ -36,7 +45,28 @@ def _write_pkg(root: Path, mark: str) -> None: (root / "shadowpkg.py").write_text(f"MARK = {mark!r}\n", encoding="utf-8") -def _run(child_py: Path, env: dict) -> str: +def _write_distribution(root: Path, version: str) -> None: + _write_pkg(root, version) + metadata = root / f"shadowpkg-{version}.dist-info" + metadata.mkdir() + (metadata / "METADATA").write_text( + f"Metadata-Version: 2.1\nName: shadowpkg\nVersion: {version}\n", + encoding="utf-8", + ) + (metadata / "top_level.txt").write_text("shadowpkg\n", encoding="utf-8") + + +def _site_packages(python: Path) -> Path: + result = subprocess.run( + [str(python), "-c", "import sysconfig; print(sysconfig.get_paths()['purelib'])"], + capture_output=True, + text=True, + check=True, + ) + return Path(result.stdout.strip()) + + +def _run(child_py: Path, env: dict[str, str]) -> str: r = subprocess.run( [str(child_py), "-c", "import shadowpkg; print('WINNER=' + shadowpkg.MARK)"], env=env, @@ -65,10 +95,8 @@ def test_host_site_packages_do_not_shadow_the_workload(tmp_path): assert _run(child_py, env) == "workload" -def test_abi_matched_cache_still_beats_the_workload(tmp_path): - """roar's ABI-matched runtime cache (~/.cache/roar/runtime//...) is still - PREPENDED — it must beat the child's wrong-ABI system copies. Here it wins over - the workload copy, confirming the must-win branch is preserved.""" +def test_abi_matched_cache_does_not_shadow_the_workload(tmp_path): + """A cache entry that overlaps the workload is not activated.""" child_py = _roarless_python(tmp_path) cache_home = tmp_path / "xdg" cache_pkg = cache_home / "roar" / "runtime" / "cp999" / "site-packages" @@ -81,4 +109,79 @@ def test_abi_matched_cache_still_beats_the_workload(tmp_path): env["ROAR_RUNTIME_PYTHONPATH"] = str(cache_pkg) env.pop("ROAR_WRAP", None) - assert _run(child_py, env) == "cache" + assert _run(child_py, env) == "workload" + + +@pytest.mark.skipif(platform.system() != "Linux", reason="product path uses Linux tracer") +def test_roar_run_records_the_distribution_that_the_workload_imported( + temp_git_repo: Path, + git_commit: Callable[[str], None], +) -> None: + """The child imports and records its own pin even when Roar's host has another. + + The workload is intentionally Roar-unaware. A separate parent venv imports this + worktree through a .pth file, while a child venv owns the package under test. + """ + test_conftest._ensure_repo_local_ptrace_tracer() + env_root = temp_git_repo.parent / f"{temp_git_repo.name}-crossenv" + parent_root = env_root / "parent" + child_root = env_root / "child" + venv.EnvBuilder(with_pip=False).create(str(parent_root)) + venv.EnvBuilder(with_pip=False).create(str(child_root)) + scripts_dir = "Scripts" if sys.platform == "win32" else "bin" + parent_python = parent_root / scripts_dir / "python" + child_python = child_root / scripts_dir / "python" + parent_site = _site_packages(parent_python) + child_site = _site_packages(child_python) + current_site = _site_packages(Path(sys.executable)) + + (parent_site / "roar-worktree.pth").write_text( + f"{SOURCE_ROOT}\n{current_site}\n", + encoding="utf-8", + ) + _write_distribution(parent_site, "9.0") + _write_distribution(child_site, "1.0") + + script = temp_git_repo / "workload.py" + script.write_text( + "import json, shadowpkg\n" + "with open('observed.json', 'w', encoding='utf-8') as handle:\n" + " json.dump({'version': shadowpkg.MARK, 'file': shadowpkg.__file__}, handle)\n", + encoding="utf-8", + ) + git_commit("add cross-environment workload") + + env = dict(os.environ) + env.pop("PYTHONPATH", None) + result = subprocess.run( + [ + str(parent_python), + "-m", + "roar", + "run", + "--tracer", + "ptrace", + "--no-tracer-fallback", + str(child_python), + script.name, + ], + cwd=temp_git_repo, + env=env, + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, f"stdout={result.stdout!r}\nstderr={result.stderr!r}" + + observed = json.loads((temp_git_repo / "observed.json").read_text(encoding="utf-8")) + assert observed["version"] == "1.0" + assert Path(observed["file"]).is_relative_to(child_site) + + connection = sqlite3.connect(temp_git_repo / ".roar" / "roar.db") + try: + row = connection.execute("SELECT metadata FROM jobs ORDER BY id DESC LIMIT 1").fetchone() + finally: + connection.close() + assert row is not None and row[0] + metadata = json.loads(row[0]) + assert metadata["packages"]["pip"]["shadowpkg"] == "1.0" From 0c0d84aa024be5a3942bb3f23e1ab30f27de2189 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Fri, 7 Aug 2026 12:34:41 +0000 Subject: [PATCH 14/52] sitecustomize: don't let roar's env shadow the workload's sys.path (P0-14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit roar makes itself importable in a traced child by putting entries on ROAR_RUNTIME_PYTHONPATH, which sitecustomize applied by prepending ALL of them. On a cross-interpreter run (roar under system 3.10, workload venv 3.12) that put roar's host dist-packages at sys.path[0], shadowing the recorded pins — the run executed against host packages. It crashed loudly only because the host's pyparsing was 5 years stale; a merely-different host package would import fine and certify GREEN for the wrong reason (and the venv-vs-manifest guard can't see it — the venv is right; the child's sys.path is wrong). Fix: split ROAR_RUNTIME_PYTHONPATH by precedence. - roar's ABI-matched runtime CACHE (~/.cache/roar/runtime//…) stays PREPENDED — it must beat the child's wrong-ABI/stale system copies (the original typing_extensions-4.15-vs-system-4.4 fix). - everything else (roar's host site-packages) is now APPENDED, so the workload's own venv always wins. roar's core injection is pure-Python, so it's still importable via the appended path; ABI-specific backend deps are handled separately by the existing runtime gate. Cache-root detection is inlined (matching lazy_install.runtime_cache_root) because this runs before roar is importable. Tests: cache entry prepended (must-win preserved); host entry appended, not at front (P0-14); early-return unchanged. Plus an integration test running a roar-less child that confirms a workload package beats a host one (non-vacuous: fails without the fix) and the cache still beats the workload. Harness note: installing roar under the recorded interpreter (`uv tool install --python `) sidesteps this by making roar's interpreter == the child's; this fix removes the silent-false-pass regardless. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../execution/runtime/inject/sitecustomize.py | 57 ++++++--- .../runtime/test_sitecustomize_path_order.py | 115 ++++++++++-------- .../test_no_crossenv_syspath_shadow.py | 84 +++++++++++++ 3 files changed, 190 insertions(+), 66 deletions(-) create mode 100644 tests/integration/test_no_crossenv_syspath_shadow.py diff --git a/roar/execution/runtime/inject/sitecustomize.py b/roar/execution/runtime/inject/sitecustomize.py index 3e44a03f..011f8725 100644 --- a/roar/execution/runtime/inject/sitecustomize.py +++ b/roar/execution/runtime/inject/sitecustomize.py @@ -5,19 +5,40 @@ import sys -def _prepend_roar_runtime_pythonpath() -> None: - """Prepend ``ROAR_RUNTIME_PYTHONPATH`` entries to ``sys.path`` (in order). - - When the traced Python has a lazy-installed ABI-matched runtime tree on - ``ROAR_RUNTIME_PYTHONPATH``, that tree must beat system site-packages — - the system copies are the wrong-ABI ones, which is exactly why we - installed the tree in the first place. Prepending the whole list in - declared order (cache, then bundled fallbacks) keeps the lazy-install - cache at ``sys.path[0]``. - - Logic is inlined (rather than imported from elsewhere in roar) because - this runs *before* roar is necessarily importable — making roar - importable is exactly what this function does. +def _roar_runtime_cache_root() -> str: + """``$XDG_CACHE_HOME/roar/runtime`` (default ``~/.cache/roar/runtime``). + + Inlined to match ``lazy_install.runtime_cache_root()`` — this runs before + roar is importable, so it can't call into roar. + """ + xdg = os.environ.get("XDG_CACHE_HOME") + base = xdg if xdg else os.path.join(os.path.expanduser("~"), ".cache") + return os.path.abspath(os.path.join(base, "roar", "runtime")) + + +def _add_roar_runtime_pythonpath() -> None: + """Make roar importable in the traced process **without letting roar's own + environment shadow the workload's recorded packages**. + + ``ROAR_RUNTIME_PYTHONPATH`` carries two very different kinds of entry: + + - roar's lazy-installed **ABI-matched runtime cache** + (``~/.cache/roar/runtime//site-packages``). This *must* beat the + system's wrong-ABI copies — that is why it was installed — so it is + **prepended**. + - roar's package root / the parent interpreter's **site-packages**, added so + a non-editable or cross-interpreter child can import roar at all. These are + **appended**, so the workload's own venv always wins. + + Prepending the second kind was **P0-14**: when roar ran under a different + interpreter than the child (e.g. roar under system 3.10, workload venv 3.12), + its host ``dist-packages`` landed at ``sys.path[0]`` and shadowed the recorded + pins — the run executed against host packages and could certify GREEN for the + wrong reason. roar's core injection is pure-Python, so appending still leaves + it importable; ABI-specific backend deps are handled separately by the runtime + gate below. + + Inlined (not imported from roar) because this runs before roar is importable. """ if importlib.util.find_spec("roar") is not None: return @@ -28,11 +49,17 @@ def _prepend_roar_runtime_pythonpath() -> None: ] if not new_paths: return - sys.path[:0] = new_paths + cache_root = _roar_runtime_cache_root() + must_win = [p for p in new_paths if os.path.abspath(p).startswith(cache_root + os.sep)] + others = [p for p in new_paths if p not in must_win] + if must_win: + sys.path[:0] = must_win # ABI-matched cache must beat wrong-ABI system copies + if others: + sys.path.extend(others) # roar's env must NOT shadow the workload's venv (P0-14) os.environ["ROAR_RUNTIME_PYTHONPATH_ACTIVE"] = os.pathsep.join(new_paths) -_prepend_roar_runtime_pythonpath() +_add_roar_runtime_pythonpath() from roar.execution.framework.runtime_imports import RuntimeImportController from roar.execution.runtime.inject.support import ( diff --git a/tests/execution/runtime/test_sitecustomize_path_order.py b/tests/execution/runtime/test_sitecustomize_path_order.py index 11830572..c2e71e4d 100644 --- a/tests/execution/runtime/test_sitecustomize_path_order.py +++ b/tests/execution/runtime/test_sitecustomize_path_order.py @@ -1,15 +1,19 @@ -"""sitecustomize prepends ROAR_RUNTIME_PYTHONPATH entries (in order). - -Behavior under test: when the traced Python doesn't already have roar -importable (the cross-Python lazy-install scenario), ``sitecustomize.py`` -must put the entries from ``ROAR_RUNTIME_PYTHONPATH`` at the *front* of -``sys.path``, preserving the declared order. Appending (the old behavior) -lets the system's stale site-packages win — which is the friction-journal -bug where lazy-installed ``typing_extensions`` 4.15.0 lost to the -system's 4.4.x. - -Tested via subprocess so we exercise the real sitecustomize module-import -side effects without polluting the test process's ``sys.path``. +"""sitecustomize places ROAR_RUNTIME_PYTHONPATH entries with the right precedence. + +When the traced Python can't already import roar (the cross-Python / +lazy-install scenario), ``sitecustomize.py`` adds ``ROAR_RUNTIME_PYTHONPATH`` +entries to ``sys.path`` with two different precedences: + +- roar's **ABI-matched runtime cache** (``~/.cache/roar/runtime//…``) is + **prepended** — it must beat the child's system copies (the friction-journal + bug: a lazy-installed ``typing_extensions`` 4.15 losing to system 4.4). +- everything else (roar's host site-packages, added only so a cross-interpreter + child can import roar) is **appended** — prepending it was P0-14: roar's host + packages shadowed the workload's recorded pins and the run executed against + host packages. + +Tested via subprocess so we exercise the real sitecustomize module-import side +effects without polluting the test process's ``sys.path``. """ from __future__ import annotations @@ -22,11 +26,21 @@ SOURCE_ROOT = Path(__file__).resolve().parents[3] +# Patches find_spec("roar") -> None so the add-path codepath runs, then imports +# sitecustomize. Callers append their own print statements. +_PATCH_AND_IMPORT = """ +import importlib.util, importlib, sys +_real = importlib.util.find_spec +importlib.util.find_spec = lambda name, *a, **k: None if name == "roar" else _real(name, *a, **k) +importlib.import_module("roar.execution.runtime.inject.sitecustomize") +""" + def _run_python( code: str, *, roar_runtime_pythonpath: str | None = None, + extra_env: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: """Run a subprocess Python with sitecustomize loaded from this source tree.""" env = dict(os.environ) @@ -38,6 +52,8 @@ def _run_python( else: env.pop("ROAR_RUNTIME_PYTHONPATH", None) env.pop("ROAR_WRAP", None) # skip the backend-dispatch gate; we only care about path order + if extra_env: + env.update(extra_env) return subprocess.run( [sys.executable, "-c", code], capture_output=True, @@ -49,61 +65,58 @@ def _run_python( ) -def test_runtime_pythonpath_entries_land_at_front_in_declared_order(tmp_path: Path) -> None: - """When roar isn't already importable, ROAR_RUNTIME_PYTHONPATH wins.""" - fake_runtime = tmp_path / "fake-runtime" - fake_runtime.mkdir() - fake_other = tmp_path / "fake-other" - fake_other.mkdir() +def test_abi_matched_cache_is_prepended(tmp_path: Path) -> None: + """roar's ABI-matched runtime cache must beat system copies -> prepended.""" + cache_home = tmp_path / "xdg" + cache_dir = cache_home / "roar" / "runtime" / "cp999" / "site-packages" + cache_dir.mkdir(parents=True) - # Force find_spec("roar") to return None by monkey-patching it before - # sitecustomize runs. We mark roar's site-packages location empty for the - # purposes of this subprocess by inserting a stub finder ahead of it that - # claims "roar is missing" — that's what triggers the prepend codepath. - code = textwrap.dedent( - """ - import importlib.util - import sys + code = _PATCH_AND_IMPORT + "print(f'first={sys.path[0]}')\n" + result = _run_python( + code, + roar_runtime_pythonpath=str(cache_dir), + extra_env={"XDG_CACHE_HOME": str(cache_home)}, + ) + assert result.returncode == 0, result.stderr + assert f"first={cache_dir}" in result.stdout, result.stdout - _real_find_spec = importlib.util.find_spec - def _patched_find_spec(name, *args, **kwargs): - if name == "roar": - return None - return _real_find_spec(name, *args, **kwargs) - importlib.util.find_spec = _patched_find_spec - import importlib - importlib.import_module("roar.execution.runtime.inject.sitecustomize") - # The prepend has run; assert the entries are at the front, in order. - print(f"first={sys.path[0]}") - print(f"second={sys.path[1]}") +def test_host_site_packages_are_appended_not_prepended(tmp_path: Path) -> None: + """P0-14: a non-cache runtime entry is appended, so it can't shadow the + workload — present on sys.path, but not at the front.""" + fake_host = tmp_path / "fake-host" + fake_host.mkdir() + + code = _PATCH_AND_IMPORT + textwrap.dedent( + f""" + target = {str(fake_host)!r} + print("present=" + str(target in sys.path)) + print("at_front=" + str(sys.path[0] == target)) + print("at_back=" + str(sys.path[-1] == target)) """ ) - result = _run_python( - code, - roar_runtime_pythonpath=os.pathsep.join([str(fake_runtime), str(fake_other)]), - ) + result = _run_python(code, roar_runtime_pythonpath=str(fake_host)) assert result.returncode == 0, result.stderr out = result.stdout - assert f"first={fake_runtime}" in out, out - assert f"second={fake_other}" in out, out + assert "present=True" in out, out + assert "at_front=False" in out, out + assert "at_back=True" in out, out -def test_no_prepend_when_roar_already_importable(tmp_path: Path) -> None: - """When roar is already importable, the function early-returns and leaves sys.path alone.""" +def test_no_change_when_roar_already_importable(tmp_path: Path) -> None: + """When roar is already importable, the function early-returns and leaves + sys.path alone.""" fake_runtime = tmp_path / "fake-runtime" fake_runtime.mkdir() code = textwrap.dedent( f""" - import importlib - import sys - # Roar IS importable (PYTHONPATH points at source root). Prepend should no-op. + import importlib, sys + # Roar IS importable (PYTHONPATH points at source root). Add-path should no-op. importlib.import_module("roar.execution.runtime.inject.sitecustomize") target = {str(fake_runtime)!r} - in_top_three = target in sys.path[:3] - print("in_top_three=" + str(in_top_three)) + print("present=" + str(target in sys.path)) """ ) result = _run_python(code, roar_runtime_pythonpath=str(fake_runtime)) assert result.returncode == 0, result.stderr - assert "in_top_three=False" in result.stdout, result.stdout + assert "present=False" in result.stdout, result.stdout diff --git a/tests/integration/test_no_crossenv_syspath_shadow.py b/tests/integration/test_no_crossenv_syspath_shadow.py new file mode 100644 index 00000000..1cf10398 --- /dev/null +++ b/tests/integration/test_no_crossenv_syspath_shadow.py @@ -0,0 +1,84 @@ +"""P0-14: roar's runtime injection must not let roar's own environment shadow the +workload's recorded packages on ``sys.path``. + +roar makes itself importable in a traced child by putting entries on +``ROAR_RUNTIME_PYTHONPATH``; ``sitecustomize`` applies them. Previously *all* of +them were prepended, so on a cross-interpreter run roar's host ``dist-packages`` +landed at ``sys.path[0]`` and shadowed the recorded pins (the run executed +against host packages, and could certify GREEN for the wrong reason). The fix: +prepend only roar's ABI-matched runtime **cache**; append everything else so the +workload's venv wins. + +These run a roar-less child interpreter with a ``sitecustomize`` on +``PYTHONPATH`` and assert which copy of a shadowed module wins. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import venv +from pathlib import Path + +INJECT_DIR = Path(__file__).resolve().parents[2] / "roar" / "execution" / "runtime" / "inject" + + +def _roarless_python(tmp_path: Path) -> Path: + """A Python that cannot already import roar (so the injection path runs).""" + child = tmp_path / "child" + venv.EnvBuilder(with_pip=False).create(str(child)) + return child / ("Scripts" if sys.platform == "win32" else "bin") / "python" + + +def _write_pkg(root: Path, mark: str) -> None: + root.mkdir(parents=True, exist_ok=True) + (root / "shadowpkg.py").write_text(f"MARK = {mark!r}\n", encoding="utf-8") + + +def _run(child_py: Path, env: dict) -> str: + r = subprocess.run( + [str(child_py), "-c", "import shadowpkg; print('WINNER=' + shadowpkg.MARK)"], + env=env, + capture_output=True, + text=True, + ) + for line in r.stdout.splitlines(): + if line.startswith("WINNER="): + return line[len("WINNER=") :] + raise AssertionError(f"no winner line.\nstdout={r.stdout!r}\nstderr={r.stderr!r}") + + +def test_host_site_packages_do_not_shadow_the_workload(tmp_path): + """A non-cache runtime entry (roar's host site-packages) is APPENDED, so the + workload's own copy (on PYTHONPATH) wins. Before the fix it was prepended and + 'host' won — a silent execution against host packages.""" + child_py = _roarless_python(tmp_path) + _write_pkg(tmp_path / "workload", "workload") + _write_pkg(tmp_path / "host", "host") + + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join([str(INJECT_DIR), str(tmp_path / "workload")]) + env["ROAR_RUNTIME_PYTHONPATH"] = str(tmp_path / "host") + env.pop("ROAR_WRAP", None) + + assert _run(child_py, env) == "workload" + + +def test_abi_matched_cache_still_beats_the_workload(tmp_path): + """roar's ABI-matched runtime cache (~/.cache/roar/runtime//...) is still + PREPENDED — it must beat the child's wrong-ABI system copies. Here it wins over + the workload copy, confirming the must-win branch is preserved.""" + child_py = _roarless_python(tmp_path) + cache_home = tmp_path / "xdg" + cache_pkg = cache_home / "roar" / "runtime" / "cp999" / "site-packages" + _write_pkg(cache_pkg, "cache") + _write_pkg(tmp_path / "workload", "workload") + + env = dict(os.environ) + env["XDG_CACHE_HOME"] = str(cache_home) + env["PYTHONPATH"] = os.pathsep.join([str(INJECT_DIR), str(tmp_path / "workload")]) + env["ROAR_RUNTIME_PYTHONPATH"] = str(cache_pkg) + env.pop("ROAR_WRAP", None) + + assert _run(child_py, env) == "cache" From 687816c9fe4538584bb07d8381696cb7303945f4 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Thu, 6 Aug 2026 21:36:32 +0000 Subject: [PATCH 15/52] tracker: attribute imported packages by name, not only by file (P0-6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `get_used_packages` attributed a package to a job only via each loaded module's `__file__`. An aliasing logging shim (`sys.modules["wandb"] = trackio`) makes `import wandb` resolve to trackio's file, so wandb was never recorded in the job's package freeze — yet the job genuinely needs it (diffusers/accelerate gate on `importlib.metadata.version("wandb")`, and its install is required). On reproduce, the wandb-less freeze was reinstalled and the workload died at `import wandb`. The record lied by omission. Fix: also attribute packages the workload imported by NAME. `tracking_import` wraps `builtins.__import__`, so `import wandb` is captured even when `sys.modules["wandb"]` is pre-populated (Python still calls `__import__("wandb", ...)`). write_log now passes `imported_modules` into `get_used_packages`, which unions in each imported top-level name's distribution. No false positives — a name is added only if the workload actually imported it AND it maps to an INSTALLED distribution (there is no unknown-name fallback on this path), and the tracer's own package (`roar`) is never attributed. A package that was never imported can never appear. Tests (tests/execution/runtime/test_used_packages_by_name.py): aliased import is attributed by name; a never-imported package is never added; an imported-but-not- installed name and roar itself are both skipped; and an end-to-end pass through the real `tracking_import` -> `write_log` -> log `used_packages`. Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/execution/runtime/inject/tracker.py | 29 +++++- .../runtime/test_used_packages_by_name.py | 91 +++++++++++++++++++ 2 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 tests/execution/runtime/test_used_packages_by_name.py diff --git a/roar/execution/runtime/inject/tracker.py b/roar/execution/runtime/inject/tracker.py index 2a06a62d..79c66fa0 100644 --- a/roar/execution/runtime/inject/tracker.py +++ b/roar/execution/runtime/inject/tracker.py @@ -68,6 +68,7 @@ def get_installed_packages() -> dict[str, str]: def get_used_packages( modules_files: Sequence[str], installed_packages: Mapping[str, str | None], + imported_modules: Sequence[str] = (), ) -> dict[str, str | None]: used: dict[str, str | None] = {} @@ -104,6 +105,30 @@ def get_used_packages( except Exception: pass + # Attribute packages the workload IMPORTED BY NAME, not just by loaded file. + # An aliased import (e.g. a `sys.modules["wandb"] = trackio` logging shim) + # leaves the loaded module's __file__ pointing at the alias target, so the + # file pass above records trackio and never wandb — yet the job genuinely + # depends on wandb (its dist metadata is queried, its install is required), + # so wandb silently drops out of the recorded environment. The import NAME is + # the honest signal: `import wandb` is captured even when sys.modules is + # pre-populated, because Python still calls __import__("wandb", ...). + # + # No false positives: we only add a name that (a) the workload actually + # imported, and (b) maps to an INSTALLED distribution — there is no + # unknown-name fallback here, and the tracer's own package is never + # attributed. A package that was never imported can never appear. + try: + for name in imported_modules: + top = name.split(".")[0] + if not top or top.startswith("_") or top == "roar": + continue + for pkg_name in pkg_dist_map.get(top, []): + if pkg_name in installed_packages and pkg_name not in used: + used[pkg_name] = installed_packages[pkg_name] + except Exception: + pass + return used @@ -254,7 +279,9 @@ def write_log(self) -> None: ) ) installed_packages = get_installed_packages() - used_packages = get_used_packages(modules_files, installed_packages) + used_packages = get_used_packages( + modules_files, installed_packages, sorted(self.imported_modules) + ) data = { "opened_files": sorted(self.opened_files), "imported_modules": sorted(self.imported_modules), diff --git a/tests/execution/runtime/test_used_packages_by_name.py b/tests/execution/runtime/test_used_packages_by_name.py new file mode 100644 index 00000000..3f3d7200 --- /dev/null +++ b/tests/execution/runtime/test_used_packages_by_name.py @@ -0,0 +1,91 @@ +"""P0-6: packages the workload imported by NAME are attributed even when the +loaded module's file points elsewhere (an aliasing logging shim, e.g. +``sys.modules["wandb"] = trackio``). Guarantee: no false positives — a package +is recorded only if it was actually imported *and* is installed; the tracer +itself is never attributed. +""" + +from __future__ import annotations + +import importlib.metadata as ilm +import json +import sys +import types + +from roar.execution.runtime.inject.tracker import ( + RuntimeInjectionTracker, + get_used_packages, +) + + +def test_imported_name_attributes_installed_dist_despite_alias(monkeypatch): + """`import wandb` while `wandb` is aliased to another module: the file pass + sees no wandb file, but the name pass records the real distribution.""" + monkeypatch.setattr(ilm, "packages_distributions", lambda: {"wandb": ["wandb"]}) + used = get_used_packages( + modules_files=[], # the aliased import loaded no wandb file + installed_packages={"wandb": "0.16.0", "trackio": "0.1.0"}, + imported_modules=["wandb"], + ) + assert used == {"wandb": "0.16.0"} + + +def test_never_imported_package_is_not_added(monkeypatch): + """No false positives: a package that was not imported is never recorded, + even though it is installed and maps to a distribution.""" + monkeypatch.setattr( + ilm, "packages_distributions", lambda: {"wandb": ["wandb"], "numpy": ["numpy"]} + ) + used = get_used_packages( + modules_files=[], + installed_packages={"wandb": "0.16.0", "numpy": "2.0.0"}, + imported_modules=["numpy", "os", "sys"], # wandb never imported + ) + assert "wandb" not in used + assert used.get("numpy") == "2.0.0" # imported and installed -> attributed + + +def test_imported_but_not_installed_and_tracer_never_attributed(monkeypatch): + """An imported name that isn't an installed dist is skipped (no unknown-name + fallback on this path), and roar (the tracer) is never recorded as a dep.""" + monkeypatch.setattr( + ilm, "packages_distributions", lambda: {"ghost": ["ghost"], "roar": ["roar-cli"]} + ) + used = get_used_packages( + modules_files=[], + installed_packages={"roar-cli": "0.4.4"}, # 'ghost' not installed + imported_modules=["ghost", "roar", "roar.execution.runtime"], + ) + assert used == {} + + +def test_shadowed_import_recorded_through_write_log(tmp_path, monkeypatch): + """End-to-end through the real capture path: tracking_import records the name, + write_log runs get_used_packages, and the shadowed package lands in the log's + used_packages — while a never-imported package would not.""" + from roar.execution.runtime.inject import tracker as tmod + + log_path = tmp_path / "log.json" + + class _Ctl: + def handle_import(self, *args, **kwargs): + return None + + tracker = RuntimeInjectionTracker( + {"ROAR_LOG_FILE": str(log_path)}, + _Ctl(), + log_file=str(log_path), + inject_dir=str(tmp_path / "inject"), + ) + monkeypatch.setattr(tmod, "get_installed_packages", lambda: {"wandb": "0.16.0"}) + monkeypatch.setattr(ilm, "packages_distributions", lambda: {"wandb": ["wandb"]}) + + # The shim: `wandb` resolves to a stand-in module with no __file__, so the + # file pass can't see it. The workload then imports it -> captured by name. + monkeypatch.setitem(sys.modules, "wandb", types.ModuleType("trackio_standin")) + tracker.tracking_import("wandb") # the real import-capture path + + tracker.write_log() + payload = json.loads(log_path.read_text()) + assert "wandb" in payload["imported_modules"] + assert payload["used_packages"].get("wandb") == "0.16.0" From 1d08b1e1e72c73cca545762b2f2a4a5c423b2ae5 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Thu, 6 Aug 2026 22:26:47 +0000 Subject: [PATCH 16/52] test(P0-6): read write_log output write-path-agnostically So the end-to-end test stays correct once P0-9's per-PID sharding lands (write_log then writes {log}. rather than the canonical path). No behavior change to the fix itself. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/execution/runtime/test_used_packages_by_name.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/execution/runtime/test_used_packages_by_name.py b/tests/execution/runtime/test_used_packages_by_name.py index 3f3d7200..543d9c9b 100644 --- a/tests/execution/runtime/test_used_packages_by_name.py +++ b/tests/execution/runtime/test_used_packages_by_name.py @@ -9,6 +9,7 @@ import importlib.metadata as ilm import json +import os import sys import types @@ -86,6 +87,10 @@ def handle_import(self, *args, **kwargs): tracker.tracking_import("wandb") # the real import-capture path tracker.write_log() - payload = json.loads(log_path.read_text()) + # write_log writes the canonical path on its own, or a per-PID shard once the + # P0-9 sharding change is present; read whichever it produced. + shard = log_path.with_name(f"{log_path.name}.{os.getpid()}") + written = log_path if log_path.exists() else shard + payload = json.loads(written.read_text()) assert "wandb" in payload["imported_modules"] assert payload["used_packages"].get("wandb") == "0.16.0" From fce9a7744ec9a306f8f439aed55d06d92af5ee8a Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Fri, 7 Aug 2026 01:43:34 +0000 Subject: [PATCH 17/52] tracker: keep roar (P0-11) and the workload's own package (P0-12) out of the freeze MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two ways a package that isn't a real third-party dependency was landing in the recorded freeze: P0-11 — roar records ITSELF. The file pass mapped site-packages/roar -> roar-cli. Harmless noise on a PyPI release (roar-cli==0.4.3 resolves) but fatal on an unpublished build: the freeze pins roar-cli==0.4.4.dev0, which can't resolve, so reproduce can never rebuild a row captured on a dev build. _install_roar installs roar-cli separately and unpinned, so the pin is always redundant. The name pass already skipped `roar`; the file pass now does too. P0-12 — a #264 regression. `pip install -e .` (and the .egg-info a later `pip uninstall` leaves behind, which importlib.metadata still reports installed) made the name pass re-pin the workload's OWN package from PyPI. The name pass now skips any dist whose metadata resolves inside the workload repo (workload_root, threaded from write_log as os.getcwd()); a genuine site-packages dep outside the repo is still recorded. Tests: roar-cli never enters the freeze via the file pass; an editable/egg-info self-package is skipped; a real out-of-repo dep is still pinned. All non-vacuous. Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/execution/runtime/inject/tracker.py | 41 ++++++++++++- .../runtime/test_used_packages_by_name.py | 57 +++++++++++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/roar/execution/runtime/inject/tracker.py b/roar/execution/runtime/inject/tracker.py index 79c66fa0..2706d515 100644 --- a/roar/execution/runtime/inject/tracker.py +++ b/roar/execution/runtime/inject/tracker.py @@ -65,12 +65,29 @@ def get_installed_packages() -> dict[str, str]: return packages +def _dist_is_in_repo(dist_name: str, repo_root: str) -> bool: + """True if ``dist_name``'s metadata resolves inside ``repo_root`` — i.e. it is + the workload's OWN package (an editable ``pip install -e .``, or the leftover + ``.egg-info`` a later ``pip uninstall`` doesn't remove), not a real + third-party dependency installed under site-packages.""" + try: + from importlib import metadata as importlib_metadata + + dist = importlib_metadata.distribution(dist_name) + path = getattr(dist, "_path", None) or dist.locate_file("") + return os.path.abspath(str(path)).startswith(repo_root + os.sep) + except Exception: + return False + + def get_used_packages( modules_files: Sequence[str], installed_packages: Mapping[str, str | None], imported_modules: Sequence[str] = (), + workload_root: str | None = None, ) -> dict[str, str | None]: used: dict[str, str | None] = {} + repo_root = os.path.abspath(workload_root) if workload_root else None try: from importlib import metadata as importlib_metadata @@ -94,6 +111,12 @@ def get_used_packages( continue if top_dir.startswith("_") or top_dir.endswith(".so"): continue + if top_dir == "roar": + # roar records itself otherwise. roar-cli is installed separately + # and unpinned by _install_roar, so the pin is always redundant — + # harmless noise on a PyPI release, but fatal on an unpublished + # build (roar-cli==X.Y.dev0 can't resolve). P0-11. + continue pkg_names = pkg_dist_map.get(top_dir, []) for pkg_name in pkg_names: @@ -124,8 +147,17 @@ def get_used_packages( if not top or top.startswith("_") or top == "roar": continue for pkg_name in pkg_dist_map.get(top, []): - if pkg_name in installed_packages and pkg_name not in used: - used[pkg_name] = installed_packages[pkg_name] + if pkg_name not in installed_packages or pkg_name in used: + continue + # Skip the workload's OWN package. `pip install -e .` (and the + # leftover .egg-info a later `pip uninstall` leaves, which + # importlib.metadata still reports as installed) resolves inside + # the repo, not site-packages. Pinning it would re-pin from PyPI + # the self-install we uninstalled to keep the tracer honest. + # P0-12 (#264 regression). + if repo_root and _dist_is_in_repo(pkg_name, repo_root): + continue + used[pkg_name] = installed_packages[pkg_name] except Exception: pass @@ -280,7 +312,10 @@ def write_log(self) -> None: ) installed_packages = get_installed_packages() used_packages = get_used_packages( - modules_files, installed_packages, sorted(self.imported_modules) + modules_files, + installed_packages, + sorted(self.imported_modules), + workload_root=os.getcwd(), ) data = { "opened_files": sorted(self.opened_files), diff --git a/tests/execution/runtime/test_used_packages_by_name.py b/tests/execution/runtime/test_used_packages_by_name.py index 543d9c9b..19dfdc24 100644 --- a/tests/execution/runtime/test_used_packages_by_name.py +++ b/tests/execution/runtime/test_used_packages_by_name.py @@ -94,3 +94,60 @@ def handle_import(self, *args, **kwargs): payload = json.loads(written.read_text()) assert "wandb" in payload["imported_modules"] assert payload["used_packages"].get("wandb") == "0.16.0" + + +def test_roar_is_never_recorded_in_the_freeze_via_file_pass(monkeypatch): + """P0-11: roar records itself otherwise. A dev build would then pin + roar-cli==X.Y.dev0, which can't resolve on reproduce. The file pass must skip + roar just like the name pass does.""" + monkeypatch.setattr(ilm, "packages_distributions", lambda: {"roar": ["roar-cli"]}) + used = get_used_packages( + modules_files=["/venv/lib/python3.12/site-packages/roar/__init__.py"], + installed_packages={"roar-cli": "0.4.4.dev0"}, + ) + assert "roar-cli" not in used and "roar" not in used + + +class _FakeDist: + def __init__(self, path): + self._path = path + + def locate_file(self, rel=""): + return self._path + + +def test_self_package_from_editable_egg_info_is_not_pinned(tmp_path, monkeypatch): + """P0-12 (#264 regression): `pip install -e .` (or its leftover + .egg-info after uninstall) resolves inside the repo, so the name pass + must not re-pin the workload's own package from PyPI.""" + repo = tmp_path / "repo" + egg_info = repo / "mypkg.egg-info" + egg_info.mkdir(parents=True) + monkeypatch.setattr(ilm, "packages_distributions", lambda: {"mypkg": ["mypkg"]}) + monkeypatch.setattr(ilm, "distribution", lambda name: _FakeDist(egg_info)) + used = get_used_packages( + modules_files=[], + installed_packages={"mypkg": "1.0"}, + imported_modules=["mypkg"], + workload_root=str(repo), + ) + assert "mypkg" not in used + + +def test_real_site_packages_dep_outside_repo_is_still_pinned(tmp_path, monkeypatch): + """The P0-12 skip must NOT drop a genuine dependency: a dist whose metadata + lives outside the repo (normal site-packages install) is still recorded even + when workload_root is set.""" + repo = tmp_path / "repo" + repo.mkdir() + dist_info = tmp_path / "site-packages" / "wandb-0.16.0.dist-info" + dist_info.mkdir(parents=True) + monkeypatch.setattr(ilm, "packages_distributions", lambda: {"wandb": ["wandb"]}) + monkeypatch.setattr(ilm, "distribution", lambda name: _FakeDist(dist_info)) + used = get_used_packages( + modules_files=[], + installed_packages={"wandb": "0.16.0"}, + imported_modules=["wandb"], + workload_root=str(repo), + ) + assert used.get("wandb") == "0.16.0" From 6a69e44cffd88ef076e20b0a61c07131535b8731 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Fri, 7 Aug 2026 13:23:26 +0000 Subject: [PATCH 18/52] tracker: scope the name pass to ALIASED imports only (P0-13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #264's name pass attributed every imported name that mapped to an installed distribution. On the HF stack that over-attributes: accelerate *probes* optional integrations (`import sagemaker`), and on a SageMaker AMI those happen to be installed, so 13 substrate packages (sagemaker-core/-train/…, transformer_engine, …) landed in the freeze — mutually unsatisfiable, so row 008's env setup died. The name pass's only legitimate job is recovering an import the file pass mis-attributed because it was ALIASED (`sys.modules["wandb"] = trackio`). Scope it to exactly that: attribute a name only when it was imported AND the module actually loaded for it lives in a *different* site-packages package than the name (detected via a new loaded_files map: name -> loaded module __file__, built from sys.modules in write_log). This keeps wandb (loaded as trackio) and drops: - normally-loaded imports (name == loaded package) -> the file pass's job; - merely-probed optional imports (loaded as themselves, or not loaded) -> P0-13; - the workload's own editable package (loaded from the repo, not site-packages). Tests: aliased import attributed; probed import NOT attributed (loaded-as-self and not-loaded, non-vacuous vs the old pass); never-imported not added; not- installed / roar / self-package excluded; end-to-end via write_log. Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/execution/runtime/inject/tracker.py | 67 +++++++--- .../runtime/test_used_packages_by_name.py | 114 +++++++++++------- 2 files changed, 119 insertions(+), 62 deletions(-) diff --git a/roar/execution/runtime/inject/tracker.py b/roar/execution/runtime/inject/tracker.py index 2706d515..8fcda763 100644 --- a/roar/execution/runtime/inject/tracker.py +++ b/roar/execution/runtime/inject/tracker.py @@ -80,14 +80,29 @@ def _dist_is_in_repo(dist_name: str, repo_root: str) -> bool: return False +def _site_packages_top(fpath: str) -> str | None: + """The top-level package dir for a file under site-packages, else None.""" + idx = fpath.find("site-packages/") + if idx < 0: + return None + top = fpath[idx + len("site-packages/") :].split("/")[0] + if top.endswith(".py"): + top = top[:-3] + if top.startswith("_") or top.endswith((".dist-info", ".egg-info", ".so")): + return None + return top + + def get_used_packages( modules_files: Sequence[str], installed_packages: Mapping[str, str | None], imported_modules: Sequence[str] = (), workload_root: str | None = None, + loaded_files: Mapping[str, str] | None = None, ) -> dict[str, str | None]: used: dict[str, str | None] = {} repo_root = os.path.abspath(workload_root) if workload_root else None + loaded = loaded_files or {} try: from importlib import metadata as importlib_metadata @@ -128,33 +143,39 @@ def get_used_packages( except Exception: pass - # Attribute packages the workload IMPORTED BY NAME, not just by loaded file. - # An aliased import (e.g. a `sys.modules["wandb"] = trackio` logging shim) - # leaves the loaded module's __file__ pointing at the alias target, so the - # file pass above records trackio and never wandb — yet the job genuinely - # depends on wandb (its dist metadata is queried, its install is required), - # so wandb silently drops out of the recorded environment. The import NAME is - # the honest signal: `import wandb` is captured even when sys.modules is - # pre-populated, because Python still calls __import__("wandb", ...). + # Recover packages the workload IMPORTED but that the file pass mis-attributed + # because the import was ALIASED — e.g. a `sys.modules["wandb"] = trackio` + # logging shim leaves the loaded module's __file__ pointing at trackio, so the + # file pass records trackio and never wandb, yet the job genuinely needs wandb + # (its dist metadata is queried; its install is required). # - # No false positives: we only add a name that (a) the workload actually - # imported, and (b) maps to an INSTALLED distribution — there is no - # unknown-name fallback here, and the tracer's own package is never - # attributed. A package that was never imported can never appear. + # Scope this strictly to the aliased case: attribute a name only when it was + # imported AND the module actually loaded for it lives in a DIFFERENT + # site-packages package than the name. This is precisely what the file pass + # cannot see. It deliberately excludes: + # - normally-loaded imports (name == loaded package) -> the file pass's job; + # - merely-probed optional imports that happen to be installed (e.g. + # accelerate probing `sagemaker` on a SageMaker AMI) -> not loaded as an + # alias, so not attributed. Attributing those poisoned the freeze with + # unsatisfiable substrate pins — P0-13 (#264 regression). + # A never-imported package can never appear; the tracer's own package and the + # workload's own (editable/self) package are excluded as well. try: for name in imported_modules: top = name.split(".")[0] if not top or top.startswith("_") or top == "roar": continue + loaded_file = loaded.get(top) + if not loaded_file: + continue # not actually loaded (find_spec probe / lazy import) + loaded_top = _site_packages_top(loaded_file) + if loaded_top is None or loaded_top == top: + continue # loaded as itself / not under site-packages -> file pass handles it for pkg_name in pkg_dist_map.get(top, []): if pkg_name not in installed_packages or pkg_name in used: continue - # Skip the workload's OWN package. `pip install -e .` (and the - # leftover .egg-info a later `pip uninstall` leaves, which - # importlib.metadata still reports as installed) resolves inside - # the repo, not site-packages. Pinning it would re-pin from PyPI - # the self-install we uninstalled to keep the tracer honest. - # P0-12 (#264 regression). + # The workload's OWN package (editable / leftover .egg-info in the + # repo) is not a third-party dep — P0-12. if repo_root and _dist_is_in_repo(pkg_name, repo_root): continue used[pkg_name] = installed_packages[pkg_name] @@ -311,11 +332,21 @@ def write_log(self) -> None: ) ) installed_packages = get_installed_packages() + # name -> loaded module file, so get_used_packages can tell an ALIASED + # import (sys.modules[name] resolves to a different package) from a normal + # or merely-probed one. Keyed by the sys.modules key (the import name), + # whose __file__ may point at the alias target. + loaded_files = { + name: os.path.abspath(getattr(module, "__file__", "")) + for name, module in sys.modules.items() + if getattr(module, "__file__", None) + } used_packages = get_used_packages( modules_files, installed_packages, sorted(self.imported_modules), workload_root=os.getcwd(), + loaded_files=loaded_files, ) data = { "opened_files": sorted(self.opened_files), diff --git a/tests/execution/runtime/test_used_packages_by_name.py b/tests/execution/runtime/test_used_packages_by_name.py index 19dfdc24..a492f6c3 100644 --- a/tests/execution/runtime/test_used_packages_by_name.py +++ b/tests/execution/runtime/test_used_packages_by_name.py @@ -1,8 +1,12 @@ -"""P0-6: packages the workload imported by NAME are attributed even when the -loaded module's file points elsewhere (an aliasing logging shim, e.g. -``sys.modules["wandb"] = trackio``). Guarantee: no false positives — a package -is recorded only if it was actually imported *and* is installed; the tracer -itself is never attributed. +"""P0-6 / P0-13: the name pass recovers a genuinely-used package the file pass +mis-attributed because the import was ALIASED (e.g. `sys.modules["wandb"] = +trackio`) — and ONLY that case. It must not attribute a normally-loaded import +(file pass's job) nor a merely-probed optional import that happens to be +installed (P0-13: `accelerate` probing `sagemaker` on a SageMaker AMI). + +Aliasing is detected via `loaded_files` (name -> the module file actually loaded +for it): a name is attributed only when its loaded module lives in a *different* +site-packages package than the name. """ from __future__ import annotations @@ -19,36 +23,62 @@ ) -def test_imported_name_attributes_installed_dist_despite_alias(monkeypatch): - """`import wandb` while `wandb` is aliased to another module: the file pass - sees no wandb file, but the name pass records the real distribution.""" +def _loaded_as(pkg: str) -> str: + """A site-packages module file for top-level package ``pkg``.""" + return f"/venv/lib/python3.12/site-packages/{pkg}/__init__.py" + + +def test_aliased_import_attributed_by_name(monkeypatch): + """`import wandb` aliased to trackio: file pass sees only trackio, but the + name pass sees wandb was imported yet loaded a *different* package -> records + wandb.""" monkeypatch.setattr(ilm, "packages_distributions", lambda: {"wandb": ["wandb"]}) used = get_used_packages( - modules_files=[], # the aliased import loaded no wandb file + modules_files=[], installed_packages={"wandb": "0.16.0", "trackio": "0.1.0"}, imported_modules=["wandb"], + loaded_files={"wandb": _loaded_as("trackio")}, # aliased ) assert used == {"wandb": "0.16.0"} -def test_never_imported_package_is_not_added(monkeypatch): - """No false positives: a package that was not imported is never recorded, - even though it is installed and maps to a distribution.""" - monkeypatch.setattr( - ilm, "packages_distributions", lambda: {"wandb": ["wandb"], "numpy": ["numpy"]} +def test_probed_optional_import_is_not_attributed(monkeypatch): + """P0-13 (#264 regression): an optional import that merely happened to be + installed (loaded as ITSELF, or not loaded at all) is not aliased, so the + name pass leaves it out — no unsatisfiable substrate in the freeze.""" + monkeypatch.setattr(ilm, "packages_distributions", lambda: {"sagemaker": ["sagemaker-core"]}) + installed = {"sagemaker-core": "2.9.0"} + # loaded as itself (a real but merely-probed import) -> file pass's job, not ours + used_self = get_used_packages( + modules_files=[], + installed_packages=installed, + imported_modules=["sagemaker"], + loaded_files={"sagemaker": _loaded_as("sagemaker")}, + ) + # probed via find_spec / lazy import -> never in loaded_files at all + used_absent = get_used_packages( + modules_files=[], + installed_packages=installed, + imported_modules=["sagemaker"], + loaded_files={}, ) + assert "sagemaker-core" not in used_self + assert "sagemaker-core" not in used_absent + + +def test_never_imported_package_is_not_added(monkeypatch): + monkeypatch.setattr(ilm, "packages_distributions", lambda: {"wandb": ["wandb"]}) used = get_used_packages( modules_files=[], - installed_packages={"wandb": "0.16.0", "numpy": "2.0.0"}, - imported_modules=["numpy", "os", "sys"], # wandb never imported + installed_packages={"wandb": "0.16.0"}, + imported_modules=["numpy", "os"], # wandb never imported + loaded_files={"wandb": _loaded_as("trackio")}, # even if (somehow) aliased ) assert "wandb" not in used - assert used.get("numpy") == "2.0.0" # imported and installed -> attributed def test_imported_but_not_installed_and_tracer_never_attributed(monkeypatch): - """An imported name that isn't an installed dist is skipped (no unknown-name - fallback on this path), and roar (the tracer) is never recorded as a dep.""" + """An aliased name that isn't installed is skipped; roar is never attributed.""" monkeypatch.setattr( ilm, "packages_distributions", lambda: {"ghost": ["ghost"], "roar": ["roar-cli"]} ) @@ -56,14 +86,15 @@ def test_imported_but_not_installed_and_tracer_never_attributed(monkeypatch): modules_files=[], installed_packages={"roar-cli": "0.4.4"}, # 'ghost' not installed imported_modules=["ghost", "roar", "roar.execution.runtime"], + loaded_files={"ghost": _loaded_as("ghost_alias"), "roar": _loaded_as("roar_rt")}, ) assert used == {} def test_shadowed_import_recorded_through_write_log(tmp_path, monkeypatch): """End-to-end through the real capture path: tracking_import records the name, - write_log runs get_used_packages, and the shadowed package lands in the log's - used_packages — while a never-imported package would not.""" + write_log builds loaded_files from sys.modules and runs get_used_packages, and + the aliased package lands in the log's used_packages.""" from roar.execution.runtime.inject import tracker as tmod log_path = tmp_path / "log.json" @@ -81,25 +112,24 @@ def handle_import(self, *args, **kwargs): monkeypatch.setattr(tmod, "get_installed_packages", lambda: {"wandb": "0.16.0"}) monkeypatch.setattr(ilm, "packages_distributions", lambda: {"wandb": ["wandb"]}) - # The shim: `wandb` resolves to a stand-in module with no __file__, so the - # file pass can't see it. The workload then imports it -> captured by name. - monkeypatch.setitem(sys.modules, "wandb", types.ModuleType("trackio_standin")) - tracker.tracking_import("wandb") # the real import-capture path + # The shim: `wandb` resolves to a stand-in whose __file__ is trackio's, so the + # file pass records trackio, not wandb — but the name pass detects the alias. + stand_in = types.ModuleType("trackio_standin") + stand_in.__file__ = _loaded_as("trackio") + monkeypatch.setitem(sys.modules, "wandb", stand_in) + tracker.tracking_import("wandb") tracker.write_log() - # write_log writes the canonical path on its own, or a per-PID shard once the - # P0-9 sharding change is present; read whichever it produced. shard = log_path.with_name(f"{log_path.name}.{os.getpid()}") - written = log_path if log_path.exists() else shard + written = log_path if log_path.exists() else shard # canonical, or per-PID shard (P0-9) payload = json.loads(written.read_text()) assert "wandb" in payload["imported_modules"] assert payload["used_packages"].get("wandb") == "0.16.0" def test_roar_is_never_recorded_in_the_freeze_via_file_pass(monkeypatch): - """P0-11: roar records itself otherwise. A dev build would then pin - roar-cli==X.Y.dev0, which can't resolve on reproduce. The file pass must skip - roar just like the name pass does.""" + """P0-11: the file pass must skip roar (roar-cli is installed separately and + unpinned; a dev build would otherwise pin an unresolvable roar-cli==X.Y.dev0).""" monkeypatch.setattr(ilm, "packages_distributions", lambda: {"roar": ["roar-cli"]}) used = get_used_packages( modules_files=["/venv/lib/python3.12/site-packages/roar/__init__.py"], @@ -116,28 +146,23 @@ def locate_file(self, rel=""): return self._path -def test_self_package_from_editable_egg_info_is_not_pinned(tmp_path, monkeypatch): - """P0-12 (#264 regression): `pip install -e .` (or its leftover - .egg-info after uninstall) resolves inside the repo, so the name pass - must not re-pin the workload's own package from PyPI.""" - repo = tmp_path / "repo" - egg_info = repo / "mypkg.egg-info" - egg_info.mkdir(parents=True) +def test_self_package_from_editable_is_not_pinned(tmp_path, monkeypatch): + """P0-12: the workload's own `pip install -e .` package loads from the repo, + not site-packages, so it isn't aliased and the name pass leaves it out.""" monkeypatch.setattr(ilm, "packages_distributions", lambda: {"mypkg": ["mypkg"]}) - monkeypatch.setattr(ilm, "distribution", lambda name: _FakeDist(egg_info)) used = get_used_packages( modules_files=[], installed_packages={"mypkg": "1.0"}, imported_modules=["mypkg"], - workload_root=str(repo), + loaded_files={"mypkg": str(tmp_path / "repo" / "mypkg" / "__init__.py")}, + workload_root=str(tmp_path / "repo"), ) assert "mypkg" not in used -def test_real_site_packages_dep_outside_repo_is_still_pinned(tmp_path, monkeypatch): - """The P0-12 skip must NOT drop a genuine dependency: a dist whose metadata - lives outside the repo (normal site-packages install) is still recorded even - when workload_root is set.""" +def test_real_aliased_dep_outside_repo_is_still_pinned(tmp_path, monkeypatch): + """A genuinely aliased dep whose metadata is outside the repo is still + recorded (the P0-12 repo skip must not drop it).""" repo = tmp_path / "repo" repo.mkdir() dist_info = tmp_path / "site-packages" / "wandb-0.16.0.dist-info" @@ -148,6 +173,7 @@ def test_real_site_packages_dep_outside_repo_is_still_pinned(tmp_path, monkeypat modules_files=[], installed_packages={"wandb": "0.16.0"}, imported_modules=["wandb"], + loaded_files={"wandb": _loaded_as("trackio")}, # aliased workload_root=str(repo), ) assert used.get("wandb") == "0.16.0" From d4dcfce3ca64364afe9d39e575048a434a024315 Mon Sep 17 00:00:00 2001 From: Trevor Basinger Date: Wed, 12 Aug 2026 19:51:42 +0000 Subject: [PATCH 19/52] feat(auth): support delegated GLaaS publishing --- roar/cli/publish_intent.py | 13 ++++++ roar/integrations/glaas/client.py | 10 ++++- roar/publish_auth.py | 58 +++++++++++++++++++++---- tests/unit/test_publish_auth_context.py | 29 +++++++++++++ tests/unit/test_publish_intent.py | 26 +++++++++++ 5 files changed, 126 insertions(+), 10 deletions(-) diff --git a/roar/cli/publish_intent.py b/roar/cli/publish_intent.py index 55be7b3a..95282227 100644 --- a/roar/cli/publish_intent.py +++ b/roar/cli/publish_intent.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from dataclasses import dataclass from pathlib import Path @@ -18,6 +19,8 @@ class PublishIntent: def _is_logged_in() -> bool: """True iff there's a usable GLaaS/TReqs session on this machine.""" + if os.environ.get("ROAR_DELEGATED_AUTH") == "1": + return True try: from ..auth_store import load_auth_state @@ -46,6 +49,16 @@ def resolve_publish_intent( Deterministic, so it's headless-safe — no interactive prompt is required to reach a default. """ + if os.environ.get("ROAR_DELEGATED_AUTH") == "1": + # The workload is not the authority for attribution or visibility. The + # agent supplies the exact frozen project policy, which must beat repo + # config and command flags just as operational redirects beat saved + # config elsewhere. + return PublishIntent( + public=os.environ.get("ROAR_DELEGATED_VISIBILITY") == "public", + anonymous=False, + ) + if anonymous: return PublishIntent(public=True, anonymous=True) diff --git a/roar/integrations/glaas/client.py b/roar/integrations/glaas/client.py index 470167b1..378047e0 100644 --- a/roar/integrations/glaas/client.py +++ b/roar/integrations/glaas/client.py @@ -202,6 +202,8 @@ def _request( def _make_auth_header(self, method: str, path: str, body: bytes | None = None) -> str | None: if self._force_anonymous: return None + if self._publish_auth.delegated_auth_available: + return None if self._publish_auth.access_token and not self._bearer_auth_rejected: return f"Bearer {self._publish_auth.access_token}" return make_auth_header(method, path, body) @@ -214,7 +216,11 @@ def _make_ssh_auth_header( return make_auth_header(method, path, body) def _can_fallback_from_bearer_to_ssh(self) -> bool: - if self._force_anonymous or self._bearer_auth_rejected: + if ( + self._force_anonymous + or self._bearer_auth_rejected + or self._publish_auth.delegated_auth_available + ): return False if not self._publish_auth.access_token: return False @@ -231,6 +237,8 @@ def probe_publish_auth(self) -> bool | None: """ if self._force_anonymous: return False + if self._publish_auth.delegated_auth_available: + return True if self._publish_auth.access_token: return True if not self.base_url: diff --git a/roar/publish_auth.py b/roar/publish_auth.py index a1c8e5d7..16e7291a 100644 --- a/roar/publish_auth.py +++ b/roar/publish_auth.py @@ -2,6 +2,7 @@ import contextvars import json +import os import urllib.error import urllib.request from dataclasses import dataclass @@ -30,6 +31,7 @@ class PublishAuthContext: db_user_id: str | None = None creator_identity: str | None = None ssh_auth_available: bool = False + delegated_auth_available: bool = False # Request-scoped carrier for the explicit --public/--private choice. The publish @@ -127,20 +129,29 @@ def load_publish_auth_context( db_user_id=None, creator_identity=None, ssh_auth_available=False, + delegated_auth_available=False, ) + delegated_auth_available = os.environ.get("ROAR_DELEGATED_AUTH") == "1" access_token = None auth_provider = None user_sub = None db_user_id = None - auth_state = load_auth_state() + # A delegated task deliberately ignores ambient workstation credentials. + # Its loopback broker adds the real upstream authorization out of process. + auth_state = None if delegated_auth_available else load_auth_state() if auth_state is not None: access_token = auth_state.access_token auth_provider = auth_state.provider user_sub = auth_state.user.sub or None db_user_id = auth_state.user.db_user_id - ssh_auth_available = _has_ssh_auth_credentials() + ssh_auth_available = False if delegated_auth_available else _has_ssh_auth_credentials() + + if delegated_auth_available: + auth_provider = "treqs-lineage-task" + user_sub = os.environ.get("ROAR_DELEGATED_USER_SUB") or None + db_user_id = os.environ.get("ROAR_DELEGATED_DB_USER_ID") or None # Proactively renew an expiring/expired bearer so register doesn't ride on a # token `roar whoami` already calls "expired" (which then reads as a bug when @@ -157,8 +168,15 @@ def load_publish_auth_context( access_token = None else: raise PublishAuthError(str(exc)) from exc - binding = None if allow_public_without_binding else _load_repo_binding(start_dir) - repo_scope = load_repo_scope(start_dir) + delegated_scope = _load_delegated_scope() if delegated_auth_available else None + binding = ( + delegated_scope + if delegated_auth_available + else None + if allow_public_without_binding + else _load_repo_binding(start_dir) + ) + repo_scope = None if delegated_auth_available else load_repo_scope(start_dir) # `allow_public_without_binding` permits a *scopeless* public publish, but it # must not discard a **public project scope** — that binding carries the org # attribution (supplier/author) the AI-BOM needs, and a public project's @@ -172,7 +190,8 @@ def load_publish_auth_context( repo_scope and repo_scope.mode == "project" and repo_scope.visibility == "public" ): repo_scope = None - if binding and not access_token and not ssh_auth_available: + has_publish_auth = bool(access_token or ssh_auth_available or delegated_auth_available) + if binding and not has_publish_auth: raise PublishAuthError( "Repo is linked to GLaaS but no global auth state is available. Run `roar login`." ) @@ -183,19 +202,21 @@ def load_publish_auth_context( } if repo_scope.project_id: binding["project_id"] = repo_scope.project_id - if not access_token and not ssh_auth_available: + if not has_publish_auth: raise PublishAuthError( "Repo is linked to GLaaS but no global auth state is available. Run `roar login`." ) - if not binding and not allow_public_without_binding and not access_token: + if not binding and not allow_public_without_binding and not has_publish_auth: raise PublishAuthError( "Private registration requires GLaaS login when no project scope is linked. " "Run `roar login`, use `roar scope use `, or rerun with --public." ) creator_identity = None - if not access_token and allow_public_without_binding: + if delegated_auth_available: + creator_identity = os.environ.get("ROAR_DELEGATED_CREATOR_IDENTITY") or None + elif not access_token and allow_public_without_binding: creator_identity, resolved_db_user_id = _load_authenticated_creator_identity() if resolved_db_user_id and not db_user_id: db_user_id = resolved_db_user_id @@ -205,7 +226,9 @@ def load_publish_auth_context( scope_request = { "owner_id": binding["owner_id"], "owner_type": binding["owner_type"], - "visibility": _scope_visibility(repo_scope, requested_public) or "private", + "visibility": binding.get("visibility") + or _scope_visibility(repo_scope, requested_public) + or "private", } project_id = binding.get("project_id") if project_id: @@ -230,9 +253,26 @@ def load_publish_auth_context( db_user_id=db_user_id, creator_identity=creator_identity, ssh_auth_available=ssh_auth_available, + delegated_auth_available=delegated_auth_available, ) +def _load_delegated_scope() -> dict[str, str]: + values = { + "owner_id": os.environ.get("ROAR_DELEGATED_OWNER_ID", "").strip(), + "owner_type": os.environ.get("ROAR_DELEGATED_OWNER_TYPE", "").strip(), + "project_id": os.environ.get("ROAR_DELEGATED_PROJECT_ID", "").strip(), + "visibility": os.environ.get("ROAR_DELEGATED_VISIBILITY", "").strip(), + } + if ( + not all(values.values()) + or values["owner_type"] not in {"user", "organization"} + or values["visibility"] not in {"public", "private"} + ): + raise PublishAuthError("Delegated GLaaS scope is missing or invalid") + return values + + def resolve_publish_creator_identity(context: PublishAuthContext) -> str: explicit_identity = _optional_string(context.creator_identity) if explicit_identity is not None: diff --git a/tests/unit/test_publish_auth_context.py b/tests/unit/test_publish_auth_context.py index a6c1d165..5e80fa4b 100644 --- a/tests/unit/test_publish_auth_context.py +++ b/tests/unit/test_publish_auth_context.py @@ -96,6 +96,35 @@ def test_private_publish_without_binding_uses_current_user_scope_request( } +def test_delegated_publish_ignores_ambient_auth_and_resolves_private_context( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("ROAR_DELEGATED_AUTH", "1") + monkeypatch.setenv("ROAR_DELEGATED_USER_SUB", "sub-123") + monkeypatch.setenv("ROAR_DELEGATED_DB_USER_ID", "user-123") + monkeypatch.setenv("ROAR_DELEGATED_CREATOR_IDENTITY", "treqs:user:sub-123") + monkeypatch.setenv("ROAR_DELEGATED_OWNER_ID", "org-123") + monkeypatch.setenv("ROAR_DELEGATED_OWNER_TYPE", "organization") + monkeypatch.setenv("ROAR_DELEGATED_PROJECT_ID", "project-456") + monkeypatch.setenv("ROAR_DELEGATED_VISIBILITY", "private") + + with patch("roar.publish_auth.load_auth_state", side_effect=AssertionError("must not load")): + context = load_publish_auth_context( + start_dir=tmp_path, + allow_public_without_binding=False, + ) + + assert context.access_token is None + assert context.delegated_auth_available + assert context.creator_identity == "treqs:user:sub-123" + assert context.scope_request == { + "owner_id": "org-123", + "owner_type": "organization", + "project_id": "project-456", + "visibility": "private", + } + + def test_public_scope_uses_current_user_public_scope_request(tmp_path: Path) -> None: config_dir = tmp_path / ".roar" config_dir.mkdir(parents=True) diff --git a/tests/unit/test_publish_intent.py b/tests/unit/test_publish_intent.py index a247a9ef..79ee7f4f 100644 --- a/tests/unit/test_publish_intent.py +++ b/tests/unit/test_publish_intent.py @@ -51,6 +51,32 @@ def test_unset_logged_in_defaults_private(): assert not out.defaulted_anonymous +def test_delegated_task_defaults_private_without_auth_file(monkeypatch): + monkeypatch.setenv("ROAR_DELEGATED_AUTH", "1") + monkeypatch.setenv("ROAR_DELEGATED_VISIBILITY", "private") + with ( + patch("roar.scope_config.load_repo_scope", return_value=None), + patch("roar.auth_store.load_auth_state", return_value=None), + patch("roar.integrations.config.config_get", return_value=False), + ): + out = resolve_publish_intent(None, False) + + assert not out.public and not out.anonymous + + +def test_delegated_task_uses_frozen_visibility_over_repo_and_flags(monkeypatch): + monkeypatch.setenv("ROAR_DELEGATED_AUTH", "1") + monkeypatch.setenv("ROAR_DELEGATED_VISIBILITY", "public") + with patch( + "roar.scope_config.load_repo_scope", + return_value=SimpleNamespace(mode="anonymous", visibility=None), + ) as load_repo_scope: + out = resolve_publish_intent(public=False, anonymous=True) + + assert out.public and not out.anonymous + load_repo_scope.assert_not_called() + + def test_unset_not_logged_in_defaults_anonymous_with_flag(): out = _resolve(scope=None, logged_in=False) assert out.public and out.anonymous From f00575bb1b84d6e87d6ce121cb7289f434dc90e8 Mon Sep 17 00:00:00 2001 From: Trevor Basinger Date: Fri, 14 Aug 2026 14:28:11 +0000 Subject: [PATCH 20/52] fix(auth): use registration sessions for delegated publishing --- roar/application/publish/session.py | 9 ++++- tests/application/publish/test_session.py | 44 +++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/roar/application/publish/session.py b/roar/application/publish/session.py index 1ee28699..ffaf1e21 100644 --- a/roar/application/publish/session.py +++ b/roar/application/publish/session.py @@ -355,10 +355,14 @@ def prepare_publish_session( publish_auth = resolved_remote_registry.publish_auth access_token = getattr(publish_auth, "access_token", None) ssh_auth_available = getattr(publish_auth, "ssh_auth_available", False) + delegated_auth_available = getattr(publish_auth, "delegated_auth_available", False) scope_request = getattr(publish_auth, "scope_request", None) has_access_token = isinstance(access_token, str) and bool(access_token.strip()) has_ssh_auth = ssh_auth_available if isinstance(ssh_auth_available, bool) else False + has_delegated_auth = ( + delegated_auth_available if isinstance(delegated_auth_available, bool) else False + ) anonymous_public_capable = ( scope_request is None @@ -388,7 +392,10 @@ def prepare_publish_session( ) should_use_registration_sessions = ( - has_access_token or has_ssh_auth or supports_anonymous_public_path + has_access_token + or has_ssh_auth + or has_delegated_auth + or supports_anonymous_public_path ) if should_use_registration_sessions: diff --git a/tests/application/publish/test_session.py b/tests/application/publish/test_session.py index 10ed7660..48ac02b6 100644 --- a/tests/application/publish/test_session.py +++ b/tests/application/publish/test_session.py @@ -234,6 +234,50 @@ def test_prepare_publish_session_creates_registration_session_with_scoped_ssh_on session_service.register.assert_not_called() +def test_prepare_publish_session_creates_registration_session_with_delegated_auth( + tmp_path: Path, +) -> None: + glaas_client = MagicMock() + glaas_client.publish_auth.access_token = None + glaas_client.publish_auth.scope_request = { + "owner_id": "owner-123", + "owner_type": "organization", + "project_id": "proj-123", + "visibility": "private", + } + glaas_client.publish_auth.ssh_auth_available = False + glaas_client.publish_auth.delegated_auth_available = True + session_service = MagicMock() + session_service.compute_session_hash.return_value = "session-hash" + session_service.create_registration_session.return_value = SessionRegistrationResult( + success=True, + session_hash="session-hash", + session_url=None, + registration_session_id="reg-session-delegated-123", + ) + + result = prepare_publish_session( + glaas_client=glaas_client, + session_service=session_service, + roar_dir=tmp_path / ".roar", + session_id=7, + git_context=_git_context(), + logger=MagicMock(), + register_with_glaas=True, + ) + + assert result == PreparedPublishSession( + session_hash="session-hash", + session_url=None, + registration_session_id="reg-session-delegated-123", + ) + session_service.create_registration_session.assert_called_once_with( + client_session_id=None, + mode=None, + ) + session_service.register.assert_not_called() + + def test_prepare_publish_session_uses_anonymous_public_registration_sessions_when_supported( tmp_path: Path, ) -> None: From f3d21ae6e20586c4b495386170fc2e9a0062095c Mon Sep 17 00:00:00 2001 From: Trevor Basinger Date: Fri, 14 Aug 2026 14:47:06 +0000 Subject: [PATCH 21/52] fix(auth): sync publish labels through registration scope --- roar/application/publish/put_execution.py | 3 + .../application/publish/register_execution.py | 2 + roar/application/publish/registration.py | 11 ++- roar/integrations/glaas/client.py | 21 ++++- .../application/publish/test_registration.py | 29 ++++++ tests/integration/fake_glaas.py | 93 +++++++++++++------ tests/integrations/glaas/test_client.py | 21 +++++ 7 files changed, 151 insertions(+), 29 deletions(-) diff --git a/roar/application/publish/put_execution.py b/roar/application/publish/put_execution.py index 962906f9..27d48737 100644 --- a/roar/application/publish/put_execution.py +++ b/roar/application/publish/put_execution.py @@ -820,6 +820,7 @@ def _put_prepared_with_registration_session( remote_job_uid=remote_put_job_uid, registration_errors=registration_errors, uploads=uploads, + registration_session_id=registration_session_id, ) composite_result_items = [ @@ -1106,6 +1107,7 @@ def _sync_put_job_labels_with_glaas( remote_job_uid: str, registration_errors: list[str], uploads: list[_UploadedArtifact] | None = None, + registration_session_id: str | None = None, ) -> None: """Sync the local current label document for the publish-time put job and its published artifacts (carrying ``roar.distribution.url``).""" @@ -1122,6 +1124,7 @@ def _sync_put_job_labels_with_glaas( jobs=[{"id": job_id, "job_uid": job_uid, "remote_job_uid": remote_job_uid}], artifacts=artifacts, errors=registration_errors, + registration_session_id=registration_session_id, ) def _link_put_job_artifacts_with_glaas( diff --git a/roar/application/publish/register_execution.py b/roar/application/publish/register_execution.py index fb78ff77..63dff918 100644 --- a/roar/application/publish/register_execution.py +++ b/roar/application/publish/register_execution.py @@ -406,6 +406,7 @@ def register_prepared_lineage( jobs=remote_registration_jobs, artifacts=label_artifacts, errors=registration_errors, + registration_session_id=registration_session_id, ) except Exception as e: return RegisterResult( @@ -426,6 +427,7 @@ def register_prepared_lineage( jobs=remote_registration_jobs, artifacts=label_artifacts, errors=registration_errors, + registration_session_id=registration_session_id, ) else: if has_lineage_composites(lineage.artifacts): diff --git a/roar/application/publish/registration.py b/roar/application/publish/registration.py index 381d8001..122e0207 100644 --- a/roar/application/publish/registration.py +++ b/roar/application/publish/registration.py @@ -468,6 +468,7 @@ def sync_publish_labels( jobs: list[dict[str, Any]], artifacts: list[dict[str, Any]], errors: list[str] | None = None, + registration_session_id: str | None = None, ) -> int: """Sync current local labels for published entities to GLaaS. @@ -490,7 +491,15 @@ def sync_publish_labels( glaas_client=glaas_client, ) - _label_result, label_error = resolved_remote_registry.sync_labels(payloads) + if registration_session_id: + _label_result, label_error = ( + resolved_remote_registry.client.sync_labels_under_registration_session( + registration_session_id, + payloads, + ) + ) + else: + _label_result, label_error = resolved_remote_registry.sync_labels(payloads) if label_error: if errors is not None: errors.append(f"Label sync failed: {label_error}") diff --git a/roar/integrations/glaas/client.py b/roar/integrations/glaas/client.py index 378047e0..f7b44d6a 100644 --- a/roar/integrations/glaas/client.py +++ b/roar/integrations/glaas/client.py @@ -688,8 +688,6 @@ def finalize_registration_session( allow_auth_fallback=False, ) error = _normalize_scope_error(self._publish_auth.scope_request, error) - if error is None and self._registration_session_mode == "anonymous_public": - self._clear_registration_session_auth() return result, error def sync_labels( @@ -701,6 +699,25 @@ def sync_labels( return {"created": 0, "updated": 0, "unchanged": 0}, None return self._request("POST", "/api/v1/labels/sync", {"labels": labels}) + def sync_labels_under_registration_session( + self, + registration_session_id: str, + labels: list[dict[str, Any]], + ) -> tuple[dict | None, str | None]: + """Sync labels only to lineage finalized by this registration session.""" + if not labels: + return {"created": 0, "updated": 0, "noops": 0}, None + result, error = self._request( + "POST", + f"/api/v1/registration-sessions/{registration_session_id}/labels/batch", + {"labels": labels}, + auth_header_value=self._registration_session_auth_header(), + allow_auth_fallback=False, + ) + if error is None and self._registration_session_mode == "anonymous_public": + self._clear_registration_session_auth() + return result, error + def reconcile_labels( self, payload: dict[str, Any], diff --git a/tests/application/publish/test_registration.py b/tests/application/publish/test_registration.py index 4be7ecec..e1c0128f 100644 --- a/tests/application/publish/test_registration.py +++ b/tests/application/publish/test_registration.py @@ -201,6 +201,35 @@ def test_sync_publish_labels_appends_error_when_sync_fails() -> None: assert errors == ["Label sync failed: permission denied"] +def test_sync_publish_labels_uses_registration_session_scoped_route() -> None: + client = MagicMock() + client.sync_labels_under_registration_session.return_value = ( + {"processed": 1, "created": 1}, + None, + ) + + with patch( + "roar.application.publish.registration.collect_label_sync_payloads", + return_value=[{"entity_type": "dag", "session_hash": "session-hash"}], + ): + sync_publish_labels( + glaas_client=client, + db_ctx=MagicMock(), + session_id=7, + session_hash="session-hash", + jobs=[{"job_uid": "job-1"}], + artifacts=[], + errors=[], + registration_session_id="reg-session-123", + ) + + client.sync_labels_under_registration_session.assert_called_once_with( + "reg-session-123", + [{"entity_type": "dag", "session_hash": "session-hash"}], + ) + client.sync_labels.assert_not_called() + + def test_sync_publish_labels_skips_empty_payloads() -> None: client = MagicMock() diff --git a/tests/integration/fake_glaas.py b/tests/integration/fake_glaas.py index 4e7dd90c..290489bb 100644 --- a/tests/integration/fake_glaas.py +++ b/tests/integration/fake_glaas.py @@ -160,6 +160,32 @@ def _record_artifacts(self, artifacts: list[dict[str, Any]]) -> None: if isinstance(digest, str) and digest: self.server.artifacts_by_digest[digest] = artifact + def _record_label_sync(self, labels: list[dict[str, Any]]) -> None: + self.server.label_syncs.append(labels) + for label in labels: + target_key = _label_target_key(label) + current = self.server.current_labels_by_target.get(target_key) + version = int(current.get("version", 0)) + 1 if isinstance(current, dict) else 1 + current_label = { + "id": f"label-{len(self.server.current_labels_by_target) + 1}", + "entityType": label.get("entity_type"), + "version": version, + "metadata": label.get("metadata") + if isinstance(label.get("metadata"), dict) + else {}, + "createdAt": "2026-01-01T00:00:00Z", + } + if label.get("entity_type") == "dag": + current_label["sessionHash"] = label.get("session_hash") + elif label.get("entity_type") == "job": + current_label["sessionHash"] = label.get("session_hash") + current_label["jobUid"] = label.get("job_uid") + elif label.get("entity_type") == "artifact": + current_label["sessionHash"] = label.get("session_hash") + current_label["artifactHash"] = label.get("artifact_hash") + self.server.current_labels_by_target[target_key] = current_label + self.server.label_history_by_target.setdefault(target_key, []).append(current_label) + def _resolve_creator_identity(self, authenticated_user: dict[str, str] | None) -> str: if not isinstance(authenticated_user, dict): return "anonymous" @@ -625,37 +651,52 @@ def do_POST(self) -> None: if self.path == "/api/v1/labels/sync": labels = payload.get("labels", []) if isinstance(labels, list): - self.server.label_syncs.append(labels) - for label in labels: - if not isinstance(label, dict): - continue - target_key = _label_target_key(label) - current = self.server.current_labels_by_target.get(target_key) - version = int(current.get("version", 0)) + 1 if isinstance(current, dict) else 1 - current_label = { - "id": f"label-{len(self.server.current_labels_by_target) + 1}", - "entityType": label.get("entity_type"), - "version": version, - "metadata": label.get("metadata") - if isinstance(label.get("metadata"), dict) - else {}, - "createdAt": "2026-01-01T00:00:00Z", - } - if label.get("entity_type") == "dag": - current_label["sessionHash"] = label.get("session_hash") - elif label.get("entity_type") == "job": - current_label["sessionHash"] = label.get("session_hash") - current_label["jobUid"] = label.get("job_uid") - elif label.get("entity_type") == "artifact": - current_label["sessionHash"] = label.get("session_hash") - current_label["artifactHash"] = label.get("artifact_hash") - self.server.current_labels_by_target[target_key] = current_label - self.server.label_history_by_target.setdefault(target_key, []).append(current_label) + self._record_label_sync([label for label in labels if isinstance(label, dict)]) self._write_json( 200, {"created": 0, "updated": 0, "unchanged": len(labels)}, ) + return + + registration_label_match = re.fullmatch( + r"/api/v1/registration-sessions/([^/]+)/labels/batch", + self.path, + ) + if registration_label_match: + registration_session_id = registration_label_match.group(1) + _authenticated_user, session_state = self._authorize_registration_session_write( + registration_session_id, + authorization, + ) + if session_state is None: + self._write_json(401, {"error": "Missing or invalid auth"}) return + lineage_hash = session_state.get("hash") + if session_state.get("status") != "closed" or not isinstance(lineage_hash, str): + self._write_json( + 400, + {"error": {"message": "Registration session must be finalized"}}, + ) + return + raw_labels = payload.get("labels", []) + labels = [ + {**label, "session_hash": lineage_hash} + for label in raw_labels + if isinstance(label, dict) + ] if isinstance(raw_labels, list) else [] + self._record_label_sync(labels) + self._write_json( + 200, + { + "registration_session_id": registration_session_id, + "hash": lineage_hash, + "processed": len(labels), + "created": len(labels), + "updated": 0, + "noops": 0, + }, + ) + return if self.path == "/api/v1/labels/reconcile": if authenticated_user is None: diff --git a/tests/integrations/glaas/test_client.py b/tests/integrations/glaas/test_client.py index 63d07d2a..a9fd91df 100644 --- a/tests/integrations/glaas/test_client.py +++ b/tests/integrations/glaas/test_client.py @@ -80,6 +80,27 @@ def test_finalize_current_user_private_scope_reports_server_support_gap() -> Non ) +def test_registration_session_label_sync_uses_scoped_route_and_auth() -> None: + client = _optional_auth_client() + client._registration_session_mode = "anonymous_public" + client._registration_session_token = "registration-token" + labels = [{"entity_type": "dag", "session_hash": "a" * 64, "metadata": {}}] + + with patch.object(client, "_request", return_value=({"processed": 1}, None)) as request: + result, error = client.sync_labels_under_registration_session("reg-123", labels) + + assert result == {"processed": 1} + assert error is None + request.assert_called_once_with( + "POST", + "/api/v1/registration-sessions/reg-123/labels/batch", + {"labels": labels}, + auth_header_value="RegistrationSession registration-token", + allow_auth_fallback=False, + ) + assert client._registration_session_token is None + + class TestGlaasClientExceptions: """Test that GlaasClient raises proper exceptions.""" From 8caed236ff887ee041e689a070847a67583d82de Mon Sep 17 00:00:00 2001 From: Trevor Basinger Date: Fri, 14 Aug 2026 17:05:04 +0000 Subject: [PATCH 22/52] style(auth): format delegated publishing paths --- roar/application/publish/session.py | 5 +---- tests/integration/fake_glaas.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/roar/application/publish/session.py b/roar/application/publish/session.py index ffaf1e21..55afd246 100644 --- a/roar/application/publish/session.py +++ b/roar/application/publish/session.py @@ -392,10 +392,7 @@ def prepare_publish_session( ) should_use_registration_sessions = ( - has_access_token - or has_ssh_auth - or has_delegated_auth - or supports_anonymous_public_path + has_access_token or has_ssh_auth or has_delegated_auth or supports_anonymous_public_path ) if should_use_registration_sessions: diff --git a/tests/integration/fake_glaas.py b/tests/integration/fake_glaas.py index 290489bb..c2b020ca 100644 --- a/tests/integration/fake_glaas.py +++ b/tests/integration/fake_glaas.py @@ -679,11 +679,15 @@ def do_POST(self) -> None: ) return raw_labels = payload.get("labels", []) - labels = [ - {**label, "session_hash": lineage_hash} - for label in raw_labels - if isinstance(label, dict) - ] if isinstance(raw_labels, list) else [] + labels = ( + [ + {**label, "session_hash": lineage_hash} + for label in raw_labels + if isinstance(label, dict) + ] + if isinstance(raw_labels, list) + else [] + ) self._record_label_sync(labels) self._write_json( 200, From 618207f4052d174002c3342fd3f72d4ed410928d Mon Sep 17 00:00:00 2001 From: Trevor Basinger Date: Tue, 18 Aug 2026 15:32:17 +0000 Subject: [PATCH 23/52] feat(publish): support delegated registration sessions --- pyproject.toml | 4 +- .../application/publish/lineage_composites.py | 2 + roar/application/publish/put_composites.py | 12 +- roar/application/publish/put_execution.py | 90 +++++++----- roar/application/publish/put_preparation.py | 6 + .../application/publish/register_execution.py | 121 +++++++++++----- .../publish/register_preparation.py | 6 + roar/application/publish/registration.py | 10 +- roar/application/publish/service.py | 3 +- roar/application/publish/session.py | 36 ++++- roar/integrations/glaas/client.py | 30 ++++ .../glaas/registration/artifact.py | 22 +-- .../glaas/registration/coordinator.py | 6 +- roar/integrations/glaas/registration/job.py | 17 ++- tests/integration/fake_glaas.py | 52 +++++++ tests/integration/test_put_cli_integration.py | 1 + tests/unit/put/test_put_service.py | 39 +++++ ...t_artifact_registration_phase3_fallback.py | 14 +- uv.lock | 135 +++++++++++++++++- 19 files changed, 493 insertions(+), 113 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 83f69f95..d4e54940 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "maturin" [project] name = "roar-cli" -version = "0.4.3" +version = "0.4.4" description = "Reproducibility and provenance tracker for ML training pipelines" authors = [ { name="TReqs Team", email="info@treqs.ai" } @@ -129,7 +129,7 @@ markers = [ "ray_contract: User-facing Ray contract tests using `roar run ray job submit ...`", "ray_diagnostic: Diagnostic Ray tests that intentionally inspect internal runtime details", ] -addopts = "-v --strict-markers -n auto --dist loadfile --ignore=tests/ebpf --ignore=tests/live_glaas --ignore=tests/benchmarks --ignore=tests/e2e --ignore=tests/integration/test_cli_startup.py --ignore=tests/execution/runtime/test_sitecustomize_perf.py --ignore-glob=tests/backends/*/e2e --ignore-glob=tests/backends/*/live" +addopts = "-v --strict-markers -n auto --dist loadfile --ignore=tests/ebpf --ignore=tests/live_glaas --ignore=tests/benchmarks --ignore=tests/e2e --ignore=tests/backends/osmo --ignore=tests/integration/test_cli_startup.py --ignore=tests/execution/runtime/test_sitecustomize_perf.py --ignore-glob=tests/backends/*/e2e --ignore-glob=tests/backends/*/live" timeout = 60 filterwarnings = [ "ignore::DeprecationWarning", diff --git a/roar/application/publish/lineage_composites.py b/roar/application/publish/lineage_composites.py index 4e3ddd90..61714861 100644 --- a/roar/application/publish/lineage_composites.py +++ b/roar/application/publish/lineage_composites.py @@ -54,6 +54,7 @@ def preregister_lineage_composites_with_glaas( registration_errors: list[str], composite_builder: Any, logger: ILogger, + registration_session_id: str | None = None, ) -> list[dict[str, Any]]: """Prepare and preregister lineage composites before batch link resolution.""" payloads = build_lineage_composite_payloads( @@ -68,6 +69,7 @@ def preregister_lineage_composites_with_glaas( payloads=payloads, registration_errors=registration_errors, logger=logger, + registration_session_id=registration_session_id, ) diff --git a/roar/application/publish/put_composites.py b/roar/application/publish/put_composites.py index 7b48fd50..19d931b9 100644 --- a/roar/application/publish/put_composites.py +++ b/roar/application/publish/put_composites.py @@ -39,6 +39,7 @@ def preregister_put_lineage_composites_with_glaas( dataset_identifiers: list[dict[str, Any]] | None, composite_builder: Any, logger: ILogger, + registration_session_id: str | None = None, ) -> list[dict[str, Any]]: """Prepare and preregister lineage composites for the put workflow.""" payloads = build_put_lineage_composite_payloads( @@ -55,6 +56,7 @@ def preregister_put_lineage_composites_with_glaas( payloads=payloads, registration_errors=registration_errors, logger=logger, + registration_session_id=registration_session_id, ) @@ -164,6 +166,7 @@ def register_put_composites_with_glaas( registration_errors: list[str], dataset_identifiers: list[dict[str, Any]] | None, logger: ILogger, + registration_session_id: str | None = None, ) -> list[dict[str, Any]]: """Register generated composite artifacts with GLaaS and persist local state.""" composite_registrations: list[dict[str, Any]] = [] @@ -186,7 +189,14 @@ def register_put_composites_with_glaas( if metadata_json is not None: payload["metadata"] = metadata_json - response = resolved_remote_registry.register_composite_artifact(payload) + response = ( + resolved_remote_registry.client.register_composite_artifact_under_registration_session( + registration_session_id, + payload, + ) + if registration_session_id + else resolved_remote_registry.register_composite_artifact(payload) + ) result, error = parse_composite_registration_response(response) composite_registration: dict[str, Any] = { diff --git a/roar/application/publish/put_execution.py b/roar/application/publish/put_execution.py index 27d48737..5fd9012f 100644 --- a/roar/application/publish/put_execution.py +++ b/roar/application/publish/put_execution.py @@ -220,6 +220,7 @@ def put_prepared( session_hash = prepared.session_hash registration_session_id = prepared.registration_session_id registration_session_mode = prepared.registration_session_mode + registration_session_status = prepared.registration_session_status git_context = prepared.git_context resolved = prepared.resolved_sources destination_type = prepared.destination_type @@ -246,6 +247,23 @@ def put_prepared( would_upload=[PutDryRunItem(path=str(r.path), exists=r.exists) for r in resolved], ) + # Delegated broker sessions use a deterministic client-session id. If a + # previous attempt reached finalize but the caller lost the response, + # create/resume returns the closed session and its authoritative receipt. + # Treat that as a completed retry before hashing or uploading anything; + # closed registration-session capabilities cannot accept more staging and + # re-uploading large artifacts would be both wasteful and misleading. + if registration_session_id and registration_session_status == "closed": + self._logger.debug( + "Put publication already finalized for registration session %s", + registration_session_id, + ) + return PutResult( + success=True, + session_hash=session_hash, + session_url=prepared.session_url, + ) + # Process each file: hash, create artifact, upload uploads: list[_UploadedArtifact] = [] composite_registrations: list[dict[str, Any]] = [] @@ -675,19 +693,56 @@ def _put_prepared_with_registration_session( composite_builder=self._composite_builder, declared=declared, ) + uploaded_artifacts = self._build_uploaded_artifacts_for_registration( + uploads, + source_type, + ) + staged_artifacts = prepare_batch_registration_artifacts( + uploaded_artifacts + lineage.artifacts, + registration_session_id, + fallback_to_hash=True, + prefer_blake3_first=True, + ) put_job_registered = False put_job_links_succeeded = False with Spinner("Publishing lineage to GLaaS...") as spin: + spin.update("Staging lineage composites...") + lineage_composite_registrations = preregister_put_lineage_composites_with_glaas( + db_ctx=self._db, + glaas_client=client, + lineage_artifacts=lineage.artifacts, + session_hash=fallback_session_hash, + registration_errors=registration_errors, + dataset_identifiers=dataset_identifiers, + composite_builder=self._composite_builder, + logger=self._logger, + registration_session_id=registration_session_id, + ) + spin.update("Staging output composites...") + composite_registrations = register_put_composites_with_glaas( + db_ctx=self._db, + glaas_client=client, + composite_results=composite_results_for_linking, + registration_errors=registration_errors, + dataset_identifiers=dataset_identifiers, + logger=self._logger, + registration_session_id=registration_session_id, + ) spin.update("Staging lineage jobs and artifacts...") registration_result = coordinator.register_lineage_under_registration_session( registration_session_id=registration_session_id, git_context=git_context, jobs=remote_lineage_jobs, + artifacts=staged_artifacts, ) registration_errors.extend(registration_result.errors) - if registration_result.jobs_failed == 0 and registration_result.links_failed == 0: + if ( + registration_result.jobs_failed == 0 + and registration_result.links_failed == 0 + and not registration_errors + ): spin.update("Staging put job...") put_job_result = coordinator.job_service.create_job_under_registration_session( command=command, @@ -757,39 +812,6 @@ def _put_prepared_with_registration_session( session_hash = finalize_result.session_hash session_url = finalize_result.session_url - spin.update("Registering lineage composites...") - lineage_composite_registrations = ( - preregister_put_lineage_composites_with_glaas( - db_ctx=self._db, - glaas_client=client, - lineage_artifacts=lineage.artifacts, - session_hash=session_hash, - registration_errors=registration_errors, - dataset_identifiers=dataset_identifiers, - composite_builder=self._composite_builder, - logger=self._logger, - ) - ) - - composite_results = build_publish_composite_results( - resolved_sources=resolved, - hashes_by_path=hashes_by_path, - session_hash=session_hash, - source_type=composite_source_type, - additional_composite_roots=additional_composite_roots, - composite_builder=self._composite_builder, - declared=declared, - ) - spin.update("Registering output composites...") - composite_registrations = register_put_composites_with_glaas( - db_ctx=self._db, - glaas_client=client, - composite_results=composite_results, - registration_errors=registration_errors, - dataset_identifiers=dataset_identifiers, - logger=self._logger, - ) - metadata_json = build_put_operation_metadata_json( message=message, destination=self._destination, diff --git a/roar/application/publish/put_preparation.py b/roar/application/publish/put_preparation.py index e6d04e69..8d4f7ba7 100644 --- a/roar/application/publish/put_preparation.py +++ b/roar/application/publish/put_preparation.py @@ -37,6 +37,7 @@ class PreparedPutExecution: composite_source_type: str | None registration_session_id: str | None = None registration_session_mode: str | None = None + registration_session_status: str | None = None dataset_identifiers: list[dict[str, Any]] = field(default_factory=list) additional_composite_roots: dict[Path, list[ResolvedSource]] = field(default_factory=dict) @@ -112,6 +113,11 @@ def prepare_put_execution( git_context=git_context, registration_session_id=publish_session.registration_session_id, registration_session_mode=publish_session.registration_session_mode, + registration_session_status=( + publish_session.registration_session_status + if isinstance(publish_session.registration_session_status, str) + else None + ), resolved_sources=resolved_sources, destination_type=destination_type, composite_source_type=composite_source_type, diff --git a/roar/application/publish/register_execution.py b/roar/application/publish/register_execution.py index 63dff918..c867c687 100644 --- a/roar/application/publish/register_execution.py +++ b/roar/application/publish/register_execution.py @@ -223,6 +223,7 @@ def register_prepared_lineage( confirm_callback: Callable[[list[str]], bool] | None, prepared: PreparedRegisterExecution, composite_leaf_hashes: frozenset[str] = frozenset(), + view_edges_by_job: dict[str, list[Any]] | None = None, ) -> RegisterResult: """Register already-collected local lineage with GLaaS. @@ -241,6 +242,7 @@ def register_prepared_lineage( session_id = prepared.session_id registration_session_id = prepared.registration_session_id registration_session_mode = prepared.registration_session_mode + registration_session_status = prepared.registration_session_status omit_filter = self.omit_filter detected_secrets: list[str] = [] @@ -297,6 +299,11 @@ def register_prepared_lineage( if registration_session_id else registration_jobs ) + if registration_session_id and view_edges_by_job: + for job in remote_registration_jobs: + local_job_uid = job.get("job_uid") + if isinstance(local_job_uid, str) and local_job_uid in view_edges_by_job: + job["_view_edges"] = view_edges_by_job[local_job_uid] if dry_run: return RegisterResult( @@ -323,11 +330,82 @@ def register_prepared_lineage( finalized_session_url = prepared.session_url finalize_failed = False already_registered = False + if registration_session_id and registration_session_status == "closed": + from ...core.interfaces.registration import BatchRegistrationResult + + already_registered = True + batch_result = BatchRegistrationResult( + session_registered=True, + jobs_created=0, + jobs_existing=len(remote_registration_jobs), + jobs_failed=0, + artifacts_registered=len(lineage.artifacts), + artifacts_failed=0, + links_created=0, + links_failed=0, + errors=[], + ) + if session_id is not None: + with create_database_context(roar_dir) as db_ctx: + batch_result.labels_synced = sync_publish_labels( + glaas_client=self.glaas_client, + db_ctx=db_ctx, + session_id=session_id, + session_hash=finalized_session_hash, + jobs=remote_registration_jobs, + artifacts=label_artifacts, + errors=registration_errors, + registration_session_id=registration_session_id, + ) + success = not registration_errors + if success and session_id is not None: + with create_database_context(roar_dir) as db_ctx: + persist_glaas_publication_mapping( + db_ctx=db_ctx, + session_id=session_id, + prepared_session_hash=session_hash, + finalized_session_hash=finalized_session_hash, + jobs=remote_registration_jobs, + ) + mark_lineage_synced( + db_ctx=db_ctx, + session_id=session_id, + jobs=lineage.jobs, + artifacts=lineage.artifacts, + ) + return RegisterResult( + success=success, + session_hash=finalized_session_hash, + session_url=finalized_session_url, + artifact_hash=artifact_hash, + jobs_registered=0, + jobs_existing=len(remote_registration_jobs), + artifacts_registered=len(lineage.artifacts), + links_created=0, + labels_synced=batch_result.labels_synced, + already_registered=already_registered, + error="; ".join(registration_errors) if registration_errors else None, + secrets_detected=detected_secrets, + secrets_redacted=bool(detected_secrets), + ) with Spinner("Publishing lineage to GLaaS...") as spin: refresh_job_artifact_references(lineage.jobs, lineage.artifacts) if registration_session_id: spin.update("Staging jobs and artifacts...") + if has_lineage_composites(lineage.artifacts): + spin.update("Staging composite artifacts...") + with create_database_context(roar_dir) as db_ctx: + composite_registrations = preregister_lineage_composites_with_glaas( + glaas_client=self.glaas_client, + db_ctx=db_ctx, + lineage_artifacts=lineage.artifacts, + session_hash=session_hash, + registration_errors=registration_errors, + composite_builder=self.composite_builder, + logger=self._logger, + registration_session_id=registration_session_id, + ) staged_artifacts = prepare_batch_registration_artifacts( lineage.artifacts, registration_session_id, # placeholder; client strips it before send @@ -382,42 +460,7 @@ def register_prepared_lineage( finalized_session_hash = finalize_result.session_hash finalized_session_url = finalize_result.session_url - if has_lineage_composites(lineage.artifacts): - spin.update("Registering composite artifacts...") - try: - with create_database_context(roar_dir) as db_ctx: - composite_registrations = ( - preregister_lineage_composites_with_glaas( - glaas_client=self.glaas_client, - db_ctx=db_ctx, - lineage_artifacts=lineage.artifacts, - session_hash=finalized_session_hash, - registration_errors=registration_errors, - composite_builder=self.composite_builder, - logger=self._logger, - ) - ) - if session_id is not None: - batch_result.labels_synced = sync_publish_labels( - glaas_client=self.glaas_client, - db_ctx=db_ctx, - session_id=session_id, - session_hash=finalized_session_hash, - jobs=remote_registration_jobs, - artifacts=label_artifacts, - errors=registration_errors, - registration_session_id=registration_session_id, - ) - except Exception as e: - return RegisterResult( - success=False, - session_hash=finalized_session_hash, - artifact_hash=artifact_hash, - error=f"Composite artifact registration failed: {e}", - secrets_detected=detected_secrets, - secrets_redacted=bool(detected_secrets), - ) - elif session_id is not None: + if session_id is not None: with create_database_context(roar_dir) as db_ctx: batch_result.labels_synced = sync_publish_labels( glaas_client=self.glaas_client, @@ -508,7 +551,11 @@ def register_prepared_lineage( total_artifacts_registered = batch_result.artifacts_registered + composite_registered success = ( - batch_result.jobs_failed == 0 and total_artifacts_failed == 0 and not finalize_failed + batch_result.jobs_failed == 0 + and batch_result.links_failed == 0 + and total_artifacts_failed == 0 + and not finalize_failed + and not registration_errors ) if success and session_id is not None: try: diff --git a/roar/application/publish/register_preparation.py b/roar/application/publish/register_preparation.py index 8c66c19c..2fe9ae34 100644 --- a/roar/application/publish/register_preparation.py +++ b/roar/application/publish/register_preparation.py @@ -28,6 +28,7 @@ class PreparedRegisterExecution: git_tag_repo_root: Path | None registration_session_id: str | None = None registration_session_mode: str | None = None + registration_session_status: str | None = None def prepare_register_execution( @@ -108,4 +109,9 @@ def prepare_register_execution( git_tag_repo_root=git_tag_repo_root, registration_session_id=publish_session.registration_session_id, registration_session_mode=publish_session.registration_session_mode, + registration_session_status=( + publish_session.registration_session_status + if isinstance(publish_session.registration_session_status, str) + else None + ), ) diff --git a/roar/application/publish/registration.py b/roar/application/publish/registration.py index 122e0207..0952bb53 100644 --- a/roar/application/publish/registration.py +++ b/roar/application/publish/registration.py @@ -414,6 +414,7 @@ def preregister_lineage_composites( payloads: list[CompositeRegistrationCandidate], registration_errors: list[str], logger: ILogger, + registration_session_id: str | None = None, ) -> list[dict[str, Any]]: """Register lineage composites before the main link phase.""" registrations: list[dict[str, Any]] = [] @@ -423,7 +424,14 @@ def preregister_lineage_composites( ) for item in payloads: - response = resolved_remote_registry.register_composite_artifact(item.payload) + response = ( + resolved_remote_registry.client.register_composite_artifact_under_registration_session( + registration_session_id, + item.payload, + ) + if registration_session_id + else resolved_remote_registry.register_composite_artifact(item.payload) + ) result, error = parse_composite_registration_response(response) registration: dict[str, Any] = { diff --git a/roar/application/publish/service.py b/roar/application/publish/service.py index 7b1363d7..5777dcad 100644 --- a/roar/application/publish/service.py +++ b/roar/application/publish/service.py @@ -738,6 +738,7 @@ def register_lineage_target(request: RegisterLineageRequest) -> RegisterLineageR confirm_callback=request.confirm_callback, prepared=prepared, composite_leaf_hashes=composite_leaf_hashes, + view_edges_by_job=view_edges_by_job, ) # Push the consumes view edges now that the jobs + the anchor composite are @@ -745,7 +746,7 @@ def register_lineage_target(request: RegisterLineageRequest) -> RegisterLineageR # under publication-scoped *remote* UIDs; translate via the mapping registration # persisted to the session metadata. Best-effort: never fails an otherwise- # successful registration. - if result.success and view_edges_by_job: + if result.success and view_edges_by_job and not prepared.registration_session_id: remote_uid_by_local = _load_remote_job_uid_mapping( roar_dir=request.roar_dir, session_id=collected_lineage.session_id ) diff --git a/roar/application/publish/session.py b/roar/application/publish/session.py index 55afd246..722f03ff 100644 --- a/roar/application/publish/session.py +++ b/roar/application/publish/session.py @@ -2,7 +2,9 @@ from __future__ import annotations +import hashlib import json +import os from dataclasses import dataclass from pathlib import Path from typing import Any, Protocol @@ -43,6 +45,20 @@ class PreparedPublishSession: session_url: str | None = None registration_session_id: str | None = None registration_session_mode: str | None = None + registration_session_status: str | None = None + + +def _delegated_client_session_id() -> str | None: + """Return a stable retry key scoped to one TReqs task capability.""" + identity = [ + os.environ.get("ROAR_DELEGATED_JOB_ID"), + os.environ.get("ROAR_DELEGATED_EXECUTION_ATTEMPT_ID"), + os.environ.get("ROAR_DELEGATED_TASK_ID"), + ] + if not all(identity): + return None + digest = hashlib.sha256("\0".join(str(value) for value in identity).encode()).hexdigest() + return f"roar-delegated-v1-{digest}" def build_canonical_session_payload( @@ -402,7 +418,7 @@ def prepare_publish_session( f" (mode={registration_session_mode})" if registration_session_mode else "", ) session_result = resolved_session_service.create_registration_session( - client_session_id=None, + client_session_id=(_delegated_client_session_id() if has_delegated_auth else None), mode=registration_session_mode, ) if not session_result.success: @@ -413,11 +429,29 @@ def prepare_publish_session( "Registration session ready: %s", session_result.registration_session_id, ) + if session_result.status == "closed" and session_result.registration_session_id: + finalized = resolved_session_service.finalize_registration_session( + registration_session_id=session_result.registration_session_id, + git_context=git_context, + ) + if not finalized.success: + raise ValueError( + "Closed registration session could not return its publication receipt: " + f"{finalized.error}" + ) + return PreparedPublishSession( + session_hash=finalized.session_hash, + session_url=finalized.session_url, + registration_session_id=session_result.registration_session_id, + registration_session_mode=session_result.registration_session_mode, + registration_session_status="closed", + ) return PreparedPublishSession( session_hash=session_hash, session_url=None, registration_session_id=session_result.registration_session_id, registration_session_mode=session_result.registration_session_mode, + registration_session_status=session_result.status, ) logger.debug("Registering session with GLaaS") diff --git a/roar/integrations/glaas/client.py b/roar/integrations/glaas/client.py index f7b44d6a..2bd0fde9 100644 --- a/roar/integrations/glaas/client.py +++ b/roar/integrations/glaas/client.py @@ -381,6 +381,21 @@ def register_composite_artifact( result, error = self._request("POST", "/api/v1/artifacts/composites", payload) return result, error + def register_composite_artifact_under_registration_session( + self, + registration_session_id: str, + payload: dict[str, Any], + ) -> tuple[dict | None, str | None]: + """Stage immutable composite metadata before session finalization.""" + body = {key: value for key, value in payload.items() if key != "session_hash"} + return self._request( + "POST", + f"/api/v1/registration-sessions/{registration_session_id}/artifacts/composites", + body, + auth_header_value=self._registration_session_auth_header(), + allow_auth_fallback=False, + ) + def get_composite_components(self, hash_prefix: str) -> tuple[dict | None, str | None]: """ Fetch stored component membership rows for a composite artifact. @@ -877,6 +892,21 @@ def register_job_view_edges( body: dict[str, Any] = {"view_edges": view_edges} return self._request("POST", f"/api/v1/jobs/{job_uid}/artifacts", body) + def register_job_view_edges_under_registration_session( + self, + registration_session_id: str, + job_uid: str, + view_edges: list[dict], + ) -> tuple[dict | None, str | None]: + """Stage view edges while both the job and composite are private.""" + return self._request( + "POST", + f"/api/v1/registration-sessions/{registration_session_id}/jobs/{job_uid}/view-edges", + {"view_edges": view_edges}, + auth_header_value=self._registration_session_auth_header(), + allow_auth_fallback=False, + ) + def register_job_inputs_under_registration_session( self, registration_session_id: str, diff --git a/roar/integrations/glaas/registration/artifact.py b/roar/integrations/glaas/registration/artifact.py index 090d9b00..67c6b026 100644 --- a/roar/integrations/glaas/registration/artifact.py +++ b/roar/integrations/glaas/registration/artifact.py @@ -360,6 +360,7 @@ def register_batch_under_registration_session( errors=errors, ) + validation_error_count = len(errors) total_success = 0 total_errors = 0 # Distinct artifacts (one per batch entry), not the server's per-hash @@ -389,25 +390,6 @@ def register_batch_under_registration_session( ) ) - # Backwards compat with glaas instances that pre-date the staged - # artifact endpoint (https://github.com/treqs-inc/glaas-api/pull/50). - # 404 on the very first batch means the server doesn't know the - # endpoint; bail out cleanly so the bearer link path's - # implicit stub-create still works as the legacy fallback. (This is - # exactly the pre-Phase-3 behavior — has the M1 bug, but doesn't - # break the register itself.) - if batch_error and "HTTP 404" in batch_error and batch_idx == 0 and total_success == 0: - self._logger.info( - "Phase 3 endpoint not present on this glaas instance (HTTP 404); " - "falling back to legacy link-implicit artifact creation. " - "Upgrade glaas-api to fix the 0-byte artifact issue." - ) - return ArtifactRegistrationResult( - success_count=0, - error_count=0, - errors=[], - ) - total_success += success_count total_errors += error_count @@ -419,7 +401,7 @@ def register_batch_under_registration_session( return ArtifactRegistrationResult( success_count=distinct_registered, - error_count=total_errors + len(errors), + error_count=total_errors + validation_error_count, errors=errors, ) diff --git a/roar/integrations/glaas/registration/coordinator.py b/roar/integrations/glaas/registration/coordinator.py index b3656da2..96ebb554 100644 --- a/roar/integrations/glaas/registration/coordinator.py +++ b/roar/integrations/glaas/registration/coordinator.py @@ -367,7 +367,10 @@ def register_lineage_under_registration_session( inputs = self._extract_staged_io_list(job, "_inputs", "_input_hashes") outputs = self._extract_staged_io_list(job, "_outputs", "_output_hashes") - if not inputs and not outputs: + view_edges = job.get("_view_edges") + if not isinstance(view_edges, list): + view_edges = [] + if not inputs and not outputs and not view_edges: continue link_result = self.job_service.link_job_artifacts_under_registration_session( @@ -375,6 +378,7 @@ def register_lineage_under_registration_session( job_uid=remote_job_uid, inputs=inputs, outputs=outputs, + view_edges=view_edges, ) if link_result.success: links_created += link_result.inputs_linked + link_result.outputs_linked diff --git a/roar/integrations/glaas/registration/job.py b/roar/integrations/glaas/registration/job.py index 7ce39729..9704398f 100644 --- a/roar/integrations/glaas/registration/job.py +++ b/roar/integrations/glaas/registration/job.py @@ -631,12 +631,15 @@ def link_job_artifacts_under_registration_session( job_uid: str, inputs: list[dict[str, Any]] | None, outputs: list[dict[str, Any]] | None, + view_edges: list[dict[str, Any]] | None = None, ) -> JobLinkResult: """Link artifacts to a staged job under a remote registration session.""" valid_inputs = self._normalize_link_artifacts(inputs or [], "input") valid_outputs = self._normalize_link_artifacts(outputs or [], "output") - if not valid_inputs and not valid_outputs: + valid_view_edges = [edge for edge in (view_edges or []) if isinstance(edge, dict)] + + if not valid_inputs and not valid_outputs and not valid_view_edges: self._logger.debug( "No staged artifacts to link for registration-session job %s", job_uid, @@ -681,6 +684,15 @@ def link_job_artifacts_under_registration_session( result.get("artifacts_registered", len(batch)) if result else len(batch) ) + if valid_view_edges: + result, error = self.client.register_job_view_edges_under_registration_session( + registration_session_id, + job_uid, + valid_view_edges, + ) + if error: + errors.append(f"view edges: {error}") + if valid_outputs: output_batches = _batch_artifacts(valid_outputs, MAX_ARTIFACTS_PER_REQUEST) for batch_idx, batch in enumerate(output_batches): @@ -718,10 +730,11 @@ def link_job_artifacts_under_registration_session( ) self._logger.debug( - "Linked staged artifacts to registration-session job %s: %d inputs, %d outputs", + "Linked staged artifacts to registration-session job %s: %d inputs, %d outputs, %d view edges", job_uid, inputs_linked, outputs_linked, + len(valid_view_edges), ) return JobLinkResult( success=True, diff --git a/tests/integration/fake_glaas.py b/tests/integration/fake_glaas.py index c2b020ca..9eac66cc 100644 --- a/tests/integration/fake_glaas.py +++ b/tests/integration/fake_glaas.py @@ -22,6 +22,7 @@ def __init__(self) -> None: self.registration_session_job_batches: list[dict[str, Any]] = [] self.registration_session_job_creates: list[dict[str, Any]] = [] self.artifact_batches: list[list[dict[str, Any]]] = [] + self.registration_session_artifact_batches: list[list[dict[str, Any]]] = [] self.auth_headers: list[dict[str, Any]] = [] self.input_links: list[dict[str, Any]] = [] self.output_links: list[dict[str, Any]] = [] @@ -34,6 +35,8 @@ def __init__(self) -> None: self.current_labels_by_target: dict[str, dict[str, Any]] = {} self.label_history_by_target: dict[str, list[dict[str, Any]]] = {} self.composite_registrations: list[dict[str, Any]] = [] + self.registration_session_composite_registrations: list[dict[str, Any]] = [] + self.registration_session_view_edges: list[dict[str, Any]] = [] self.artifacts_by_digest: dict[str, dict[str, Any]] = {} self.artifact_dags_by_digest: dict[str, dict[str, Any]] = {} self.session_reproductions_by_hash: dict[str, dict[str, Any]] = {} @@ -648,6 +651,51 @@ def do_POST(self) -> None: self._write_json(200, {"created": len(artifacts), "existing": 0}) return + registration_artifact_match = re.fullmatch( + r"/api/v1/registration-sessions/([^/]+)/artifacts/batch", + self.path, + ) + if registration_artifact_match: + registration_session_id = registration_artifact_match.group(1) + _authenticated_user, session_state = self._authorize_registration_session_write( + registration_session_id, + authorization, + ) + if session_state is None or session_state.get("status") != "active": + self._write_json(401, {"error": "Missing, invalid, or closed session"}) + return + artifacts = payload.get("artifacts", []) + if isinstance(artifacts, list): + self.server.registration_session_artifact_batches.append(artifacts) + self._record_artifacts(artifacts) + self._write_json(200, {"created": len(artifacts), "existing": 0}) + return + + registration_composite_match = re.fullmatch( + r"/api/v1/registration-sessions/([^/]+)/artifacts/composites", + self.path, + ) + if registration_composite_match: + registration_session_id = registration_composite_match.group(1) + _authenticated_user, session_state = self._authorize_registration_session_write( + registration_session_id, + authorization, + ) + if session_state is None or session_state.get("status") != "active": + self._write_json(401, {"error": "Missing, invalid, or closed session"}) + return + self.server.registration_session_composite_registrations.append(payload) + self._record_artifacts([payload]) + self._write_json( + 200, + { + "artifact_id": "registration-composite-" + f"{len(self.server.registration_session_composite_registrations)}", + "created": True, + }, + ) + return + if self.path == "/api/v1/labels/sync": labels = payload.get("labels", []) if isinstance(labels, list): @@ -1323,6 +1371,10 @@ def registration_session_job_creates(self) -> list[dict[str, Any]]: def artifact_batches(self) -> list[list[dict[str, Any]]]: return self._server.artifact_batches + @property + def registration_session_artifact_batches(self) -> list[list[dict[str, Any]]]: + return self._server.registration_session_artifact_batches + @property def auth_headers(self) -> list[dict[str, Any]]: return self._server.auth_headers diff --git a/tests/integration/test_put_cli_integration.py b/tests/integration/test_put_cli_integration.py index 3f5dabc7..c3865624 100644 --- a/tests/integration/test_put_cli_integration.py +++ b/tests/integration/test_put_cli_integration.py @@ -142,6 +142,7 @@ def test_put_registers_lineage_with_fake_glaas_and_updates_local_dag( assert len(fake_glaas_publish_server.job_batches) == 0 assert len(fake_glaas_publish_server.job_creates) == 0 assert len(fake_glaas_publish_server.artifact_batches) == 0 + assert len(fake_glaas_publish_server.registration_session_artifact_batches) == 1 assert len(fake_glaas_publish_server.registration_session_job_batches) == 1 assert len(fake_glaas_publish_server.registration_session_job_creates) == 1 assert fake_glaas_publish_server.registration_session_input_links diff --git a/tests/unit/put/test_put_service.py b/tests/unit/put/test_put_service.py index 9aac41c3..c810a8f8 100644 --- a/tests/unit/put/test_put_service.py +++ b/tests/unit/put/test_put_service.py @@ -83,6 +83,8 @@ def _prepared_put( session_url: str = "https://glaas.ai/dag/session_hash_abc123", destination_type: str = "memory", composite_source_type: str | None = None, + registration_session_id: str | None = None, + registration_session_status: str | None = None, ) -> PreparedPutExecution: resolved: list[ResolvedSource] = [] for source in sources: @@ -120,10 +122,47 @@ def _prepared_put( resolved_sources=resolved, destination_type=destination_type, composite_source_type=composite_source_type, + registration_session_id=registration_session_id, + registration_session_status=registration_session_status, ) class TestPutService: + def test_closed_registration_session_retry_does_not_upload_again(self, tmp_path: Path) -> None: + model_file = tmp_path / "model.pt" + model_file.write_bytes(b"model data") + db = _create_mock_db() + backend = MemoryBackend(bucket="test-bucket", prefix="models") + service = PutService( + db_context=db, + backend=backend, + destination="memory://test-bucket/models", + repo_root=tmp_path, + lineage_collector=MagicMock(), + registration_coordinator=_create_mock_coordinator(), + ) + + with patch.object(service, "_hash_files_batch") as hash_files: + result = service.put_prepared( + prepared=_prepared_put( + tmp_path, + sources=[model_file], + session_hash="authoritative-session-hash", + session_url="https://glaas.example/dag/authoritative-session-hash", + registration_session_id="registration-session-1", + registration_session_status="closed", + ), + sources=[str(model_file)], + message="retry publish model", + ) + + assert result.success is True + assert result.session_hash == "authoritative-session-hash" + assert result.session_url == "https://glaas.example/dag/authoritative-session-hash" + assert result.uploaded_files == [] + hash_files.assert_not_called() + db.jobs.create.assert_not_called() + def test_put_prepared_single_file_creates_job(self, tmp_path: Path) -> None: model_file = tmp_path / "model.pt" model_file.write_bytes(b"model data") diff --git a/tests/unit/test_artifact_registration_phase3_fallback.py b/tests/unit/test_artifact_registration_phase3_fallback.py index a34811b3..09206e1e 100644 --- a/tests/unit/test_artifact_registration_phase3_fallback.py +++ b/tests/unit/test_artifact_registration_phase3_fallback.py @@ -1,6 +1,4 @@ -"""Tests for ArtifactRegistrationService.register_batch_under_registration_session -backwards-compat fallback when talking to a glaas instance that doesn't have the -staged-artifact endpoint (https://github.com/treqs-inc/glaas-api/pull/50).""" +"""Fail-closed tests for scoped artifact staging.""" from unittest.mock import MagicMock @@ -24,10 +22,8 @@ def _artifacts(n=2): ] -def test_404_on_first_batch_silently_skips_phase3(): - """Old glaas without the staged endpoint returns 404; coordinator should - treat it as 'fall back to legacy link-implicit creation' and surface no - error, so `roar register` doesn't fail on every old server.""" +def test_404_on_first_batch_fails_closed(): + """A receiver missing the scoped endpoint is not broker-compatible.""" service, client = _service() client.register_artifacts_batch_under_registration_session.return_value = ( 0, @@ -38,8 +34,8 @@ def test_404_on_first_batch_silently_skips_phase3(): result = service.register_batch_under_registration_session(_artifacts(2), "reg-sess-x") assert result.success_count == 0 - assert result.error_count == 0 - assert result.errors == [] + assert result.error_count == 2 + assert any("404" in error for error in result.errors) def test_non_404_error_still_surfaces(): diff --git a/uv.lock b/uv.lock index 5597e453..c7b61c3e 100644 --- a/uv.lock +++ b/uv.lock @@ -15,6 +15,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] +[[package]] +name = "anyio" +version = "4.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, +] + [[package]] name = "blake3" version = "1.0.8" @@ -320,14 +334,14 @@ wheels = [ [[package]] name = "click" -version = "8.3.1" +version = "8.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] [[package]] @@ -550,6 +564,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, ] +[[package]] +name = "filelock" +version = "3.32.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/64/a02e6765de08964ed371eca577870593245afc9dfac16d037de7c10d18e6/filelock-3.32.3.tar.gz", hash = "sha256:0ffa185a3540854c95caa7fa76b76cb219d907415e2c5dc9af25fd970563487f", size = 218135, upload-time = "2026-08-13T16:00:05.577Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/8e/50f46a9c0ce8d2861a394c1347caae037ea0431d2f67d7feb151cbc4649a/filelock-3.32.3-py3-none-any.whl", hash = "sha256:7f0ca4bcc0e181c60dbbd8aa9ab5b120ebb99e4e064e83636340056f833a1f09", size = 98901, upload-time = "2026-08-13T16:00:03.974Z" }, +] + +[[package]] +name = "fsspec" +version = "2026.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, +] + [[package]] name = "google-api-core" version = "2.29.0" @@ -718,6 +750,87 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4f/dc/041be1dff9f23dac5f48a43323cd0789cb798342011c19a248d9c9335536/greenlet-3.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c10513330af5b8ae16f023e8ddbfb486ab355d04467c4679c5cfe4659975dd9", size = 1676034, upload-time = "2025-12-04T14:27:33.531Z" }, ] +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "hf-xet" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/ab/522a2ab67f27971a9d48ca666d4fca85ef7d5282d142e31fd087e27b1bbe/hf_xet-1.6.0.tar.gz", hash = "sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef", size = 920527, upload-time = "2026-08-03T22:33:13.243Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/62/3c062f593bd92ef4e77a0ef39541e3d82a0a1d3947c8a777a02a13a27828/hf_xet-1.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d", size = 4074584, upload-time = "2026-08-03T22:32:47.364Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1e/c0ad437dd267a8e435bef594acf781bbc3874ff0b6435b4962d03ecf7cc4/hf_xet-1.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675", size = 3867381, upload-time = "2026-08-03T22:32:49.049Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ee/7c0d7b6ab336167531b1c30af2af003f054af4c749becbd7209ae33a77c3/hf_xet-1.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b", size = 4453982, upload-time = "2026-08-03T22:32:50.568Z" }, + { url = "https://files.pythonhosted.org/packages/63/06/ad8eab1c9525246650cbaa821caa3cdbaca734ab1a5b8c91bea09cbd8d69/hf_xet-1.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522", size = 4249445, upload-time = "2026-08-03T22:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/d8/26/1eee8aedb0dafc1ab9717dc9ac602cde33361b232dc06803f1f6ed18b58c/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e", size = 4451099, upload-time = "2026-08-03T22:32:54.114Z" }, + { url = "https://files.pythonhosted.org/packages/67/57/0b88af1f194ab6c9c650547d9cc06bfeaab836ae4dcdb331676bfb8be95a/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9", size = 4664712, upload-time = "2026-08-03T22:32:55.547Z" }, + { url = "https://files.pythonhosted.org/packages/53/a0/26b717a9d1840e8abf48dcec64b5ed8fbe472671d38ad28d30e147132b33/hf_xet-1.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338", size = 4025906, upload-time = "2026-08-03T22:32:57.391Z" }, + { url = "https://files.pythonhosted.org/packages/49/f6/4a9966633c6fef83af997e2cff68ec1963676d412bdfd096df2a93b8e185/hf_xet-1.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765", size = 3849221, upload-time = "2026-08-03T22:32:59.123Z" }, + { url = "https://files.pythonhosted.org/packages/a2/50/7afa2c9c787405864fc47a0d1bbc02c62e9101947ed43c1f43899fc7d91d/hf_xet-1.6.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d", size = 4071729, upload-time = "2026-08-03T22:33:00.721Z" }, + { url = "https://files.pythonhosted.org/packages/4b/69/55b8dcf636142ae660fec1869fcac14c4da2e8412e14d6eee1523be77e9f/hf_xet-1.6.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a", size = 3876287, upload-time = "2026-08-03T22:33:02.251Z" }, + { url = "https://files.pythonhosted.org/packages/67/4e/a28359bf1c1ecf11eba22123168c138698f7cb576ac678f5a2e16cd5da08/hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f", size = 4464663, upload-time = "2026-08-03T22:33:03.802Z" }, + { url = "https://files.pythonhosted.org/packages/9a/69/1f0cbc2fb22ae6082d094f743d1b8945a3f36f6089cb95f42b7ee348cda7/hf_xet-1.6.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7", size = 4262538, upload-time = "2026-08-03T22:33:05.287Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3a/4f4f2301ade26e404462d3336fa11f7958d914cabbabdd6e03c3c5d5658c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb", size = 4460520, upload-time = "2026-08-03T22:33:06.81Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/311725e2a905534dfee2dcb5b08414f249147f1f12252bfc2bd24caa075c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c", size = 4675937, upload-time = "2026-08-03T22:33:08.616Z" }, + { url = "https://files.pythonhosted.org/packages/98/b7/8c59a66d15205024662f1d66968136f13893f96df1ddc5087e2e281fc95f/hf_xet-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b", size = 4033128, upload-time = "2026-08-03T22:33:10.171Z" }, + { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "huggingface-hub" +version = "1.28.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/ae/222a91937ebee7f62c0ca8f5ee0afd97577caf24c0abb927d1f5c7e9f6d2/huggingface_hub-1.28.0.tar.gz", hash = "sha256:46a2e950c09234de54093d587d1675382f0d08dbd600d9fb599b5932f5b2c6cb", size = 959609, upload-time = "2026-08-18T12:27:15.101Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/0e/eafef18f1a75e125e68395db21131db0cf868a128ecd2fce69b4df6c584b/huggingface_hub-1.28.0-py3-none-any.whl", hash = "sha256:58a8bacb03072edfc38067065e9dc24bbb34805410fcd36a1632de0b329660bb", size = 793202, upload-time = "2026-08-18T12:27:12.719Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -1429,7 +1542,7 @@ wheels = [ [[package]] name = "roar-cli" -version = "0.3.7" +version = "0.4.4" source = { editable = "." } dependencies = [ { name = "blake3" }, @@ -1450,6 +1563,7 @@ dependencies = [ dev = [ { name = "boto3" }, { name = "google-cloud-storage" }, + { name = "huggingface-hub" }, { name = "mypy" }, { name = "pytest" }, { name = "pytest-cov" }, @@ -1467,6 +1581,7 @@ requires-dist = [ { name = "cryptography", specifier = ">=42.0.0" }, { name = "dependency-injector", specifier = ">=4.40.0" }, { name = "google-cloud-storage", marker = "extra == 'dev'", specifier = ">=2.10.0" }, + { name = "huggingface-hub", marker = "extra == 'dev'", specifier = ">=0.20.0" }, { name = "msgpack", specifier = ">=1.0.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.13.0" }, { name = "pydantic", specifier = ">=2.0.0" }, @@ -1659,6 +1774,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/b8/0135fadc89e73be292b473cb820b4f5a08197779206b33191e801feeae40/tomli-2.3.0-py3-none-any.whl", hash = "sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b", size = 14408, upload-time = "2025-10-08T22:01:46.04Z" }, ] +[[package]] +name = "tqdm" +version = "4.70.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/3b/6c24bec5be5e743ffd99576daa5cc077722fc7d5bbc00bd133fa0c698dc6/tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220", size = 795438, upload-time = "2026-07-27T11:33:15.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953", size = 80184, upload-time = "2026-07-27T11:33:13.167Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" From 45714afcfacea5033ca2b59d938fa2225c505991 Mon Sep 17 00:00:00 2001 From: Trevor Basinger Date: Tue, 18 Aug 2026 16:19:49 +0000 Subject: [PATCH 24/52] fix(publish): finalize scoped existing bindings --- roar/application/publish/collection.py | 28 ++++ .../application/publish/register_execution.py | 84 +++++++++++- roar/application/publish/remote_job_uids.py | 21 +++ roar/application/publish/requests.py | 4 +- roar/application/publish/service.py | 13 +- roar/cli/commands/register.py | 28 ++-- roar/core/interfaces/registration.py | 3 + roar/integrations/glaas/client.py | 1 + .../glaas/registration/coordinator.py | 3 + tests/application/publish/test_collection.py | 36 ++++++ tests/unit/test_coordinator.py | 8 +- tests/unit/test_register_cli.py | 14 +- tests/unit/test_register_service.py | 122 ++++++++++++++++++ 13 files changed, 333 insertions(+), 32 deletions(-) diff --git a/roar/application/publish/collection.py b/roar/application/publish/collection.py index 7e6800f1..b02ade57 100644 --- a/roar/application/publish/collection.py +++ b/roar/application/publish/collection.py @@ -44,6 +44,11 @@ def collect_register_lineage( dry_run: bool = False, ) -> tuple[CollectedRegisterLineage | None, str | None]: """Collect local lineage for a resolved register target.""" + if target.kind == "active_session": + return _collect_active_session_lineage( + roar_dir=roar_dir, + lineage_collector=lineage_collector, + ) if target.kind == "step_reference": return _collect_step_lineage( step_reference=target.value, @@ -81,6 +86,29 @@ def collect_register_lineage( return None, f"Unsupported register target type: {target.kind}" +def _collect_active_session_lineage( + *, + roar_dir: Path, + lineage_collector: LineageCollector, +) -> tuple[CollectedRegisterLineage | None, str | None]: + with create_database_context(roar_dir) as db_ctx: + session = db_ctx.sessions.get_active() + if not session: + return None, "No active session. Run 'roar run' to create a session first." + session_id = int(session["id"]) + lineage = lineage_collector.collect_session(session_id, roar_dir) + + return ( + CollectedRegisterLineage( + lineage=lineage, + session_id=session_id, + artifact_hash="", + session_hash_override=None, + ), + None, + ) + + def _collect_step_lineage( *, step_reference: str, diff --git a/roar/application/publish/register_execution.py b/roar/application/publish/register_execution.py index c867c687..a3775ba9 100644 --- a/roar/application/publish/register_execution.py +++ b/roar/application/publish/register_execution.py @@ -23,7 +23,10 @@ normalize_jobs_for_registration, order_jobs_for_registration, ) -from .remote_job_uids import prepare_jobs_for_remote_publication +from .remote_job_uids import ( + apply_remote_publication_job_uid_mapping, + prepare_jobs_for_remote_publication, +) from .secrets import ( detect_lineage_secrets, filter_git_context_secrets, @@ -294,10 +297,30 @@ def register_prepared_lineage( registration_jobs = order_jobs_for_registration( normalize_jobs_for_registration(lineage.jobs) ) - remote_registration_jobs = ( - prepare_jobs_for_remote_publication(registration_jobs, session_hash) - if registration_session_id - else registration_jobs + closed_remote_uid_mapping: dict[str, str] = {} + if registration_session_id and registration_session_status == "closed" and session_id: + with create_database_context(roar_dir) as db_ctx: + closed_remote_uid_mapping = load_glaas_publication_job_mapping( + db_ctx=db_ctx, + session_id=session_id, + ) + if registration_session_id and closed_remote_uid_mapping: + remote_registration_jobs = apply_remote_publication_job_uid_mapping( + registration_jobs, + closed_remote_uid_mapping, + ) + elif registration_session_id: + remote_registration_jobs = prepare_jobs_for_remote_publication( + registration_jobs, + session_hash, + ) + else: + remote_registration_jobs = registration_jobs + missing_closed_remote_uid_mapping = bool( + registration_session_id + and registration_session_status == "closed" + and session_id + and not closed_remote_uid_mapping ) if registration_session_id and view_edges_by_job: for job in remote_registration_jobs: @@ -326,6 +349,10 @@ def register_prepared_lineage( composite_registrations: list[dict[str, Any]] = [] registration_errors: list[str] = [] + if missing_closed_remote_uid_mapping: + registration_errors.append( + "Closed publication is missing its persisted remote job identity mapping" + ) finalized_session_hash = session_hash finalized_session_url = prepared.session_url finalize_failed = False @@ -345,7 +372,7 @@ def register_prepared_lineage( links_failed=0, errors=[], ) - if session_id is not None: + if session_id is not None and not registration_errors: with create_database_context(roar_dir) as db_ctx: batch_result.labels_synced = sync_publish_labels( glaas_client=self.glaas_client, @@ -427,6 +454,24 @@ def register_prepared_lineage( already_registered = True finalized_session_hash = batch_result.already_registered_session_hash finalized_session_url = None + if batch_result.existing_binding_prepared: + finalize_result = ( + self.coordinator.session_service.finalize_registration_session( + registration_session_id=registration_session_id, + git_context=git_context, + ) + ) + if not finalize_result.success: + registration_errors.append( + "Existing publication binding finalize failed: " + f"{finalize_result.error}" + ) + elif finalize_result.session_hash != finalized_session_hash: + registration_errors.append( + "Existing publication binding returned a different lineage hash" + ) + else: + finalized_session_url = finalize_result.session_url if session_id is not None: with create_database_context(roar_dir) as db_ctx: batch_result.labels_synced = sync_publish_labels( @@ -437,6 +482,11 @@ def register_prepared_lineage( jobs=remote_registration_jobs, artifacts=label_artifacts, errors=registration_errors, + registration_session_id=( + registration_session_id + if batch_result.existing_binding_prepared + else None + ), ) elif batch_result.jobs_failed == 0 and batch_result.links_failed == 0: spin.update("Finalizing lineage...") @@ -663,6 +713,28 @@ def persist_glaas_publication_mapping( ) +def load_glaas_publication_job_mapping( + *, + db_ctx: Any, + session_id: int, +) -> dict[str, str]: + """Load the authoritative local-to-remote job mapping for label refresh.""" + session = db_ctx.sessions.get(session_id) + if not isinstance(session, dict): + return {} + metadata = _load_session_metadata(session.get("metadata")) + jobs = (((metadata.get("roar") or {}).get("remote_publication") or {}).get("glaas") or {}).get( + "jobs" + ) + if not isinstance(jobs, dict): + return {} + return { + str(local_uid): str(remote_uid) + for local_uid, remote_uid in jobs.items() + if isinstance(remote_uid, str) and remote_uid + } + + def _load_session_metadata(raw_metadata: Any) -> dict[str, Any]: if isinstance(raw_metadata, dict): return dict(raw_metadata) diff --git a/roar/application/publish/remote_job_uids.py b/roar/application/publish/remote_job_uids.py index 296cb0d1..9684a4b1 100644 --- a/roar/application/publish/remote_job_uids.py +++ b/roar/application/publish/remote_job_uids.py @@ -48,3 +48,24 @@ def prepare_jobs_for_remote_publication( prepared_jobs.append(prepared) return prepared_jobs + + +def apply_remote_publication_job_uid_mapping( + jobs: list[dict[str, Any]], + remote_uid_by_local_uid: dict[str, str], +) -> list[dict[str, Any]]: + """Apply the authoritative mapping persisted by a completed publication.""" + prepared_jobs: list[dict[str, Any]] = [] + for job in jobs: + prepared = dict(job) + local_job_uid = prepared.get("job_uid") + if isinstance(local_job_uid, str) and local_job_uid: + remote_job_uid = remote_uid_by_local_uid.get(local_job_uid) + if remote_job_uid: + prepared["remote_job_uid"] = remote_job_uid + + parent_job_uid = prepared.get("parent_job_uid") + if isinstance(parent_job_uid, str) and parent_job_uid: + prepared["remote_parent_job_uid"] = remote_uid_by_local_uid.get(parent_job_uid) + prepared_jobs.append(prepared) + return prepared_jobs diff --git a/roar/application/publish/requests.py b/roar/application/publish/requests.py index ce9eda2c..50228250 100644 --- a/roar/application/publish/requests.py +++ b/roar/application/publish/requests.py @@ -11,7 +11,9 @@ class RegisterLineageRequest: """Application request for `roar register`.""" - target: str + # None means the whole active session. Keep that intent explicit instead + # of freezing a pre-bootstrap canonical hash in the CLI. + target: str | None roar_dir: Path cwd: Path dry_run: bool = False diff --git a/roar/application/publish/service.py b/roar/application/publish/service.py index 5777dcad..d3b35d52 100644 --- a/roar/application/publish/service.py +++ b/roar/application/publish/service.py @@ -21,6 +21,7 @@ PutUploadedFile, RegisterLineageResponse, ) +from .targets import ResolvedRegisterTarget if TYPE_CHECKING: from ...db.query_context import QueryDatabaseContext @@ -566,10 +567,14 @@ def register_lineage_target(request: RegisterLineageRequest) -> RegisterLineageR ) try: - resolved_target = resolve_register_lineage_target( - request.target, - cwd=request.cwd, - roar_dir=request.roar_dir, + resolved_target = ( + ResolvedRegisterTarget(kind="active_session", value="") + if request.target is None + else resolve_register_lineage_target( + request.target, + cwd=request.cwd, + roar_dir=request.roar_dir, + ) ) runtime_kwargs: dict[str, Any] = { "start_dir": str(request.cwd), diff --git a/roar/cli/commands/register.py b/roar/cli/commands/register.py index 086fd6a1..5ce924c7 100644 --- a/roar/cli/commands/register.py +++ b/roar/cli/commands/register.py @@ -171,7 +171,7 @@ def _render_tag_summary(summary: RegisterTagSummary | None) -> None: def _apply_register_binds( ctx: RoarContext, *, - target: str, + target: str | None, response: RegisterLineageResponse, bind_targets: tuple[str, ...], no_bind: bool, @@ -190,7 +190,7 @@ def _apply_register_binds( rather than failing the command. """ refs: list[str] = [] - if not no_bind and response.artifact_hash: + if target is not None and not no_bind and response.artifact_hash: resolved_target = resolve_register_lineage_target( target, cwd=ctx.cwd, roar_dir=ctx.roar_dir ) @@ -487,15 +487,16 @@ def register( raise click.ClickException("--anonymous requires public visibility; remove --private.") target_was_defaulted = target is None + active_session_hash: str | None = None if target is None: - # No target -> register the whole active session. Resolving to the - # session's canonical hash routes through the session_hash collection - # path, which includes every job in the session (e.g. a downstream - # evaluate step), not just an artifact's upstream ancestry. + # Resolve a hash only for the confirmation preview. The application + # receives target=None so it selects the active session after publish + # bootstrap; bootstrap can change the creator identity and therefore + # the canonical hash. from ...application.query.status import StatusQueryError, compute_active_session_hash try: - target = compute_active_session_hash(ctx.roar_dir) + active_session_hash = compute_active_session_hash(ctx.roar_dir) except StatusQueryError as exc: raise click.ClickException(str(exc)) from exc @@ -510,7 +511,7 @@ def register( and not yes and not dry_run and not confirm_defaulted_active_session_publish( - session_hash=target, + session_hash=active_session_hash or "", command_name="roar register", start_dir=str(ctx.cwd), roar_dir=ctx.roar_dir, @@ -571,6 +572,7 @@ def register( web_url = _resolve_glaas_web_url(start_dir=str(ctx.cwd)) session_preview = _preview_hash(response.session_hash) if response.session_hash else "" session_url = _display_session_url(response.session_url, web_url, response.session_hash) + display_target = target if target is not None else "active session" # Apply the implicit binds up front so their result can be folded into the # register checklist (rather than printed as a separate trailing block). @@ -584,7 +586,7 @@ def register( # Format output if dry_run: - click.echo(f"Dry run: would register lineage for: {target}") + click.echo(f"Dry run: would register lineage for: {display_target}") click.echo(f" Session: {session_preview}") click.echo(f" Jobs: {response.jobs_registered}") click.echo(f" Artifacts: {response.artifacts_registered}") @@ -592,7 +594,7 @@ def register( # Secrets now ride as a line on the reproducibility punchlist below # ("no secrets in published lineage", note: none detected / N redacted). # Preview reproducibility BEFORE publishing (not yet on GLaaS). - _render_register_checklist(ctx, target, response, on_glaas=False, dry_run=True) + _render_register_checklist(ctx, display_target, response, on_glaas=False, dry_run=True) click.echo("") click.echo("GLaaS:") click.echo(f" Session: {session_url}") @@ -604,7 +606,7 @@ def register( for warning in response.warnings: click.echo(f"Warning: {warning}", err=True) _render_tag_summary(response.tag_summary) - click.echo(f"Already registered on GLaaS: {target}") + click.echo(f"Already registered on GLaaS: {display_target}") click.echo(f" Session: {session_preview}") click.echo(f" Labels: {response.labels_synced}") click.echo("") @@ -616,7 +618,7 @@ def register( else: for warning in response.warnings: click.echo(f"Warning: {warning}", err=True) - click.echo(f"Registered lineage for: {target}") + click.echo(f"Registered lineage for: {display_target}") click.echo(f" Session: {session_preview}") # Secrets ride as a punchlist line (see the checklist below). @@ -630,7 +632,7 @@ def register( # One punchlist: reproducibility checks + what register did (tag/push/ # counts + bind folded in), replacing the old separate stat/tag/bind blocks. _render_register_checklist( - ctx, target, response, on_glaas=True, bind_summaries=bind_summaries + ctx, display_target, response, on_glaas=True, bind_summaries=bind_summaries ) click.echo("") diff --git a/roar/core/interfaces/registration.py b/roar/core/interfaces/registration.py index e4cec771..b94a5f56 100644 --- a/roar/core/interfaces/registration.py +++ b/roar/core/interfaces/registration.py @@ -91,6 +91,9 @@ class BatchRegistrationResult: # (a full re-register): the existing DAG hash. The caller skips finalize and # reuses this hash instead of failing on duplicate jobs. already_registered_session_hash: str | None = None + # True only when GLaaS has server-bound the fresh registration session to + # that existing hash after exact-scope and complete-job-set verification. + existing_binding_prepared: bool = False @runtime_checkable diff --git a/roar/integrations/glaas/client.py b/roar/integrations/glaas/client.py index 2bd0fde9..804a7c30 100644 --- a/roar/integrations/glaas/client.py +++ b/roar/integrations/glaas/client.py @@ -850,6 +850,7 @@ def register_jobs_batch_under_registration_session( "already_registered": [ str(h) for h in (result.get("already_registered_session_hashes") or []) if h ], + "existing_binding_prepared": bool(result.get("existing_binding_prepared", False)), } return result.get("job_ids", []), result.get("errors", []), None, counts diff --git a/roar/integrations/glaas/registration/coordinator.py b/roar/integrations/glaas/registration/coordinator.py index 96ebb554..11bd9d9e 100644 --- a/roar/integrations/glaas/registration/coordinator.py +++ b/roar/integrations/glaas/registration/coordinator.py @@ -311,6 +311,9 @@ def register_lineage_under_registration_session( links_failed=0, errors=[], already_registered_session_hash=distinct[0], + existing_binding_prepared=bool( + batch_counts.get("existing_binding_prepared", False) + ), ) # Partial overlap (some new + some already elsewhere) or jobs # spanning multiple DAGs — can't form one DAG without a job diff --git a/tests/application/publish/test_collection.py b/tests/application/publish/test_collection.py index bc1428ca..ea588617 100644 --- a/tests/application/publish/test_collection.py +++ b/tests/application/publish/test_collection.py @@ -27,6 +27,42 @@ def test_collect_register_lineage_returns_missing_file_error(tmp_path: Path) -> assert error == "File not found: missing.csv" +def test_collect_register_lineage_selects_active_session_without_a_prebootstrap_hash( + tmp_path: Path, +) -> None: + collector = MagicMock() + collector.collect_session.return_value = LineageData( + jobs=[{"job_uid": "job-active"}], + artifacts=[], + artifact_hashes=set(), + pipeline={"id": 17}, + ) + with patch("roar.application.publish.collection.create_database_context") as mock_ctx: + db_ctx = MagicMock() + db_ctx.__enter__ = MagicMock(return_value=db_ctx) + db_ctx.__exit__ = MagicMock(return_value=None) + db_ctx.sessions.get_active.return_value = {"id": 17} + mock_ctx.return_value = db_ctx + + collected, error = collect_register_lineage( + target=ResolvedRegisterTarget(kind="active_session", value=""), + roar_dir=tmp_path / ".roar", + cwd=tmp_path, + lineage_collector=collector, + session_service=MagicMock(), + logger=MagicMock(), + ) + + assert error is None + assert collected == CollectedRegisterLineage( + lineage=collector.collect_session.return_value, + session_id=17, + artifact_hash="", + session_hash_override=None, + ) + collector.collect_session.assert_called_once_with(17, tmp_path / ".roar") + + def test_collect_register_lineage_resolves_s3_artifact_by_tracked_path(tmp_path: Path) -> None: collector = MagicMock() collector.collect.return_value = LineageData( diff --git a/tests/unit/test_coordinator.py b/tests/unit/test_coordinator.py index 790f4027..697bc476 100644 --- a/tests/unit/test_coordinator.py +++ b/tests/unit/test_coordinator.py @@ -398,7 +398,12 @@ def _reg_session_coordinator(batch_counts): def test_full_re_register_short_circuits_to_existing_dag() -> None: coordinator, artifact_service, job_service = _reg_session_coordinator( - {"created": 0, "existing": 0, "already_registered": ["dag-hash-abc"]} + { + "created": 0, + "existing": 0, + "already_registered": ["dag-hash-abc"], + "existing_binding_prepared": True, + } ) result = coordinator.register_lineage_under_registration_session( registration_session_id="reg-1", @@ -407,6 +412,7 @@ def test_full_re_register_short_circuits_to_existing_dag() -> None: artifacts=[{"hashes": [{"algorithm": "blake3", "digest": "in1"}], "size": 1}], ) assert result.already_registered_session_hash == "dag-hash-abc" + assert result.existing_binding_prepared is True assert result.session_registered is True assert result.jobs_failed == 0 # No staging / linking for an already-registered lineage. diff --git a/tests/unit/test_register_cli.py b/tests/unit/test_register_cli.py index 69f256b7..9af94ba7 100644 --- a/tests/unit/test_register_cli.py +++ b/tests/unit/test_register_cli.py @@ -363,9 +363,9 @@ def test_register_cli_renders_warnings_above_summary(tmp_path: Path) -> None: def test_register_cli_no_target_defaults_to_active_session(tmp_path: Path) -> None: """`roar register` with no target registers the whole active session. - It resolves the active session's canonical hash and passes it as the target - so the session_hash collection path runs (the full DAG, incl. downstream - steps), not an artifact's upstream-only ancestry. + It resolves the active session's canonical hash only for confirmation, then + preserves target=None so the application selects the active session after + publish bootstrap (the full DAG, including downstream steps). """ runner = CliRunner() session_hash = "c" * 64 @@ -381,7 +381,7 @@ def test_register_cli_no_target_defaults_to_active_session(tmp_path: Path) -> No assert result.exit_code == 0, result.output request = mock_register.call_args.args[0] - assert request.target == session_hash + assert request.target is None def test_register_cli_no_target_without_active_session_errors(tmp_path: Path) -> None: @@ -482,7 +482,7 @@ def test_register_cli_accepts_defaulted_active_session_publish_prompt(tmp_path: assert result.exit_code == 0, result.output request = mock_register.call_args.args[0] - assert request.target == session_hash + assert request.target is None def test_register_cli_defaulted_active_session_prompt_has_no_in_flight_warning_by_default( @@ -562,7 +562,7 @@ def test_register_cli_yes_skips_defaulted_active_session_prompt(tmp_path: Path) assert result.exit_code == 0, result.output assert "Publish the whole active session?" not in result.output request = mock_register.call_args.args[0] - assert request.target == session_hash + assert request.target is None def test_register_cli_dry_run_skips_defaulted_active_session_prompt(tmp_path: Path) -> None: @@ -583,7 +583,7 @@ def test_register_cli_dry_run_skips_defaulted_active_session_prompt(tmp_path: Pa assert result.exit_code == 0, result.output assert "Publish the whole active session?" not in result.output request = mock_register.call_args.args[0] - assert request.target == session_hash + assert request.target is None assert request.dry_run is True diff --git a/tests/unit/test_register_service.py b/tests/unit/test_register_service.py index 0d5e9880..828b352a 100644 --- a/tests/unit/test_register_service.py +++ b/tests/unit/test_register_service.py @@ -1,5 +1,6 @@ """Focused unit tests for RegisterService registration mechanics.""" +import json from pathlib import Path from unittest.mock import MagicMock, patch @@ -91,6 +92,57 @@ def test_order_jobs_for_registration_puts_parent_before_child(self) -> None: assert [job["job_uid"] for job in ordered] == ["parent-uid", "child-uid"] + def test_closed_delegated_publication_reuses_persisted_remote_job_uids( + self, tmp_path: Path + ) -> None: + remote_uid = "remote-job-authoritative" + prepared = PreparedRegisterExecution( + git_context=_git_context(tmp_path), + session_id=1, + session_hash="f" * 64, + session_url="https://glaas.example/dag/existing", + git_tag_name=None, + git_tag_repo_root=None, + registration_session_id="rs-closed", + registration_session_status="closed", + ) + with ( + patch("roar.application.publish.register_execution.config_get", return_value=False), + patch( + "roar.application.publish.register_execution.create_database_context" + ) as mock_ctx, + patch( + "roar.application.publish.register_execution.sync_publish_labels", + return_value=1, + ) as sync_labels, + ): + db_ctx = MagicMock() + db_ctx.__enter__ = MagicMock(return_value=db_ctx) + db_ctx.__exit__ = MagicMock(return_value=None) + db_ctx.sessions.get.return_value = { + "metadata": json.dumps( + {"roar": {"remote_publication": {"glaas": {"jobs": {"job-local": remote_uid}}}}} + ) + } + mock_ctx.return_value = db_ctx + + result = self.service.register_prepared_lineage( + lineage=_lineage_data(jobs=[{"id": 1, "job_uid": "job-local"}]), + roar_dir=tmp_path / ".roar", + artifact_hash="", + dry_run=False, + as_blake3=False, + skip_confirmation=True, + confirm_callback=None, + prepared=prepared, + ) + + assert result.success is True + assert result.labels_synced == 1 + synced_jobs = sync_labels.call_args.kwargs["jobs"] + assert synced_jobs[0]["job_uid"] == "job-local" + assert synced_jobs[0]["remote_job_uid"] == remote_uid + def test_normalize_jobs_for_registration_filters_known_ray_noise_jobs(self) -> None: submit_job = { "id": 1, @@ -463,3 +515,73 @@ def test_register_prepared_lineage_sends_redacted_git_context(self, tmp_path: Pa == "https://user:[REDACTED]@github.com/org/repo.git" ) assert "supersecrettoken123" not in str(finalize_call) + + def test_existing_delegated_binding_is_finalized_before_scoped_label_sync( + self, tmp_path: Path + ) -> None: + from roar.core.interfaces.registration import SessionRegistrationResult + + existing_hash = "e" * 64 + mock_coordinator = MagicMock() + mock_coordinator.register_lineage_under_registration_session.return_value = ( + BatchRegistrationResult( + session_registered=True, + jobs_created=0, + jobs_failed=0, + artifacts_registered=0, + artifacts_failed=0, + links_created=0, + links_failed=0, + errors=[], + already_registered_session_hash=existing_hash, + existing_binding_prepared=True, + ) + ) + mock_coordinator.session_service.finalize_registration_session.return_value = ( + SessionRegistrationResult( + success=True, + session_hash=existing_hash, + session_url=f"https://glaas.example/dag/{existing_hash}", + ) + ) + service = RegisterService(glaas_client=MagicMock(), coordinator=mock_coordinator) + prepared = PreparedRegisterExecution( + git_context=GitContext(repo=None, commit=None, branch=None), + session_id=None, + session_hash="local-hash", + session_url=None, + git_tag_name=None, + git_tag_repo_root=None, + registration_session_id="rs-existing", + ) + + with patch("roar.application.publish.register_execution.config_get", return_value=False): + result = service.register_prepared_lineage( + lineage=_lineage_data( + jobs=[ + { + "id": 1, + "job_uid": "job-existing", + "step_number": 1, + "timestamp": 10.0, + "command": "python train.py", + } + ], + artifacts=[], + artifact_hashes=set(), + ), + roar_dir=tmp_path / ".roar", + artifact_hash=None, + dry_run=False, + as_blake3=False, + skip_confirmation=True, + confirm_callback=None, + prepared=prepared, + ) + + assert result.success is True + assert result.session_hash == existing_hash + mock_coordinator.session_service.finalize_registration_session.assert_called_once_with( + registration_session_id="rs-existing", + git_context=prepared.git_context, + ) From ed81abca6b0dbde47d9471593a6493813319a3e0 Mon Sep 17 00:00:00 2001 From: Trevor Basinger Date: Tue, 18 Aug 2026 16:49:19 +0000 Subject: [PATCH 25/52] fix(publish): scope delegated retries to one operation --- roar/application/publish/put_execution.py | 6 +- roar/application/publish/put_preparation.py | 60 ++++++++++++++++--- roar/application/publish/service.py | 8 +++ roar/application/publish/session.py | 33 ++++++++-- .../publish/test_put_preparation.py | 60 ++++++++++++++++++- tests/application/publish/test_session.py | 32 +++++++++- 6 files changed, 182 insertions(+), 17 deletions(-) diff --git a/roar/application/publish/put_execution.py b/roar/application/publish/put_execution.py index 5fd9012f..dfe01e5c 100644 --- a/roar/application/publish/put_execution.py +++ b/roar/application/publish/put_execution.py @@ -268,8 +268,10 @@ def put_prepared( uploads: list[_UploadedArtifact] = [] composite_registrations: list[dict[str, Any]] = [] lineage_composite_registrations: list[dict[str, Any]] = [] - with Spinner(f"Hashing {len(resolved)} file(s)..."): - hashes_by_path = self._hash_files_batch([source.path for source in resolved]) + hashes_by_path = prepared.source_hashes + if not hashes_by_path: + with Spinner(f"Hashing {len(resolved)} file(s)..."): + hashes_by_path = self._hash_files_batch([source.path for source in resolved]) # Uploads are the long pole of a put (multi-GB artifacts to S3/GCS); show a # live N/M + cumulative-bytes counter rather than a dead terminal. diff --git a/roar/application/publish/put_preparation.py b/roar/application/publish/put_preparation.py index 8d4f7ba7..aa43bba9 100644 --- a/roar/application/publish/put_preparation.py +++ b/roar/application/publish/put_preparation.py @@ -2,13 +2,17 @@ from __future__ import annotations +import hashlib +import json +import os from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Mapping from urllib.parse import urlparse from ...core.interfaces.logger import ILogger from ...core.interfaces.registration import GitContext +from ...db.hashing import hash_files_blake3 from ...integrations.glaas import GlaasClient from ..git import resolve_roar_git_context from .datasets import ( @@ -35,6 +39,7 @@ class PreparedPutExecution: resolved_sources: list[ResolvedSource] destination_type: str composite_source_type: str | None + source_hashes: dict[str, str] = field(default_factory=dict) registration_session_id: str | None = None registration_session_mode: str | None = None registration_session_status: str | None = None @@ -52,6 +57,7 @@ def prepare_put_execution( destination: str, git_commit: str | None, logger: ILogger, + operation_options: Mapping[str, Any] | None = None, ) -> PreparedPutExecution: """Resolve the local context needed to execute a put workflow.""" from .source_resolution import SourceResolver @@ -76,6 +82,48 @@ def prepare_put_execution( session_service=runtime.session_service, registration_coordinator=runtime_dict.get("registration_coordinator"), ) + resolver = SourceResolver( + repo_root=repo_root, + session_repo=db_ctx.sessions, + job_repo=db_ctx.jobs, + ) + resolved_sources = resolver.resolve(sources) + source_hashes = hash_files_blake3([source.path for source in resolved_sources]) + missing_hashes = [ + str(source.path) for source in resolved_sources if str(source.path) not in source_hashes + ] + if missing_hashes: + raise OSError(f"Failed to hash put source: {missing_hashes[0]}") + + operation_payload = { + "destination": destination, + "local_session_hash": runtime.session_service.compute_session_hash( + roar_dir=str(roar_dir), + session_id=session_id, + ), + "local_session_id": session_id, + "options": dict(operation_options or {}), + "sources": sorted( + [ + { + "digest": source_hashes[str(source.path)], + "path": os.path.relpath(source.path.resolve(), repo_root.resolve()), + "relative_key": source.relative_key, + "size": source.path.stat().st_size, + } + for source in resolved_sources + ], + key=lambda source: (source["path"], source["relative_key"]), + ), + } + operation_fingerprint = hashlib.sha256( + json.dumps( + operation_payload, + sort_keys=True, + separators=(",", ":"), + ).encode() + ).hexdigest() + publish_session = prepare_publish_session( remote_registry=remote_registry, roar_dir=roar_dir, @@ -83,14 +131,9 @@ def prepare_put_execution( git_context=git_context, logger=logger, register_with_glaas=True, + operation_kind="put", + operation_fingerprint=operation_fingerprint, ) - - resolver = SourceResolver( - repo_root=repo_root, - session_repo=db_ctx.sessions, - job_repo=db_ctx.jobs, - ) - resolved_sources = resolver.resolve(sources) dataset_identifiers = infer_publish_dataset_identifiers( repo_root=repo_root, source_specs=sources, @@ -121,6 +164,7 @@ def prepare_put_execution( resolved_sources=resolved_sources, destination_type=destination_type, composite_source_type=composite_source_type, + source_hashes=source_hashes, dataset_identifiers=dataset_identifiers, additional_composite_roots=additional_composite_roots, ) diff --git a/roar/application/publish/service.py b/roar/application/publish/service.py index d3b35d52..68e9a90e 100644 --- a/roar/application/publish/service.py +++ b/roar/application/publish/service.py @@ -887,6 +887,14 @@ def put_artifacts(request: PutRequest) -> PutResponse: destination=request.destination, git_commit=git_commit, logger=logger, + operation_options={ + "anonymous": request.anonymous, + "as_dataset": request.as_dataset, + "message": request.message, + "no_tag": request.no_tag, + "public": request.public, + "step_name": request.step_name, + }, ) # Reproducibility facts for the receipt: same commit-span source as diff --git a/roar/application/publish/session.py b/roar/application/publish/session.py index 722f03ff..ef38fe54 100644 --- a/roar/application/publish/session.py +++ b/roar/application/publish/session.py @@ -48,8 +48,12 @@ class PreparedPublishSession: registration_session_status: str | None = None -def _delegated_client_session_id() -> str | None: - """Return a stable retry key scoped to one TReqs task capability.""" +def delegated_client_session_id( + *, + operation_kind: str, + operation_fingerprint: str, +) -> str | None: + """Return a stable retry key for one publication operation in a TReqs task.""" identity = [ os.environ.get("ROAR_DELEGATED_JOB_ID"), os.environ.get("ROAR_DELEGATED_EXECUTION_ATTEMPT_ID"), @@ -57,8 +61,18 @@ def _delegated_client_session_id() -> str | None: ] if not all(identity): return None - digest = hashlib.sha256("\0".join(str(value) for value in identity).encode()).hexdigest() - return f"roar-delegated-v1-{digest}" + if not operation_kind or not operation_fingerprint: + raise ValueError("Delegated publication requires an operation identity") + digest = hashlib.sha256( + "\0".join( + [ + *(str(value) for value in identity), + operation_kind, + operation_fingerprint, + ] + ).encode() + ).hexdigest() + return f"roar-delegated-v2-{digest}" def build_canonical_session_payload( @@ -320,6 +334,8 @@ def prepare_publish_session( session_hash_override: str | None = None, lineage: LineageData | None = None, creator_identity: str | None = None, + operation_kind: str = "register", + operation_fingerprint: str | None = None, ) -> PreparedPublishSession: """Compute and optionally register the publish session.""" resolved_remote_registry = coerce_remote_registry( @@ -418,7 +434,14 @@ def prepare_publish_session( f" (mode={registration_session_mode})" if registration_session_mode else "", ) session_result = resolved_session_service.create_registration_session( - client_session_id=(_delegated_client_session_id() if has_delegated_auth else None), + client_session_id=( + delegated_client_session_id( + operation_kind=operation_kind, + operation_fingerprint=operation_fingerprint or session_hash, + ) + if has_delegated_auth + else None + ), mode=registration_session_mode, ) if not session_result.success: diff --git a/tests/application/publish/test_put_preparation.py b/tests/application/publish/test_put_preparation.py index 619f2708..96d598b5 100644 --- a/tests/application/publish/test_put_preparation.py +++ b/tests/application/publish/test_put_preparation.py @@ -34,6 +34,7 @@ def test_prepare_put_execution_builds_session_git_and_source_plan(tmp_path: Path db_ctx = MagicMock() db_ctx.sessions.get_active.return_value = {"id": 7} runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" prepared_session = MagicMock( session_hash="session-hash", session_url="https://glaas/session", @@ -51,7 +52,7 @@ def test_prepare_put_execution_builds_session_git_and_source_plan(tmp_path: Path patch( "roar.application.publish.put_preparation.prepare_publish_session", return_value=prepared_session, - ), + ) as prepare_session, patch( "roar.application.publish.put_preparation.infer_publish_dataset_identifiers", return_value=[], @@ -81,8 +82,65 @@ def test_prepare_put_execution_builds_session_git_and_source_plan(tmp_path: Path resolved_sources=prepared.resolved_sources, destination_type="memory", composite_source_type=None, + source_hashes=prepared.source_hashes, ) assert [item.path for item in prepared.resolved_sources] == [model.resolve()] + assert prepared.source_hashes[str(model.resolve())] + call = prepare_session.call_args.kwargs + assert call["operation_kind"] == "put" + assert len(call["operation_fingerprint"]) == 64 + + +def test_prepare_put_execution_fingerprints_source_content(tmp_path: Path) -> None: + model = tmp_path / "model.pt" + model.write_bytes(b"model-v1") + db_ctx = MagicMock() + db_ctx.sessions.get_active.return_value = {"id": 7} + runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" + prepared_session = MagicMock( + session_hash="session-hash", + session_url=None, + registration_session_id=None, + registration_session_mode=None, + ) + + with ( + patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=GitContext(repo="repo", branch="main", commit="deadbeef"), + ), + patch( + "roar.application.publish.put_preparation.prepare_publish_session", + return_value=prepared_session, + ) as prepare_session, + ): + prepare_put_execution( + db_ctx=db_ctx, + runtime=runtime, + roar_dir=tmp_path / ".roar", + repo_root=tmp_path, + sources=["model.pt"], + destination="memory://bucket/prefix", + git_commit="deadbeef", + logger=MagicMock(), + ) + first_fingerprint = prepare_session.call_args.kwargs["operation_fingerprint"] + + model.write_bytes(b"model-v2") + prepare_put_execution( + db_ctx=db_ctx, + runtime=runtime, + roar_dir=tmp_path / ".roar", + repo_root=tmp_path, + sources=["model.pt"], + destination="memory://bucket/prefix", + git_commit="deadbeef", + logger=MagicMock(), + ) + second_fingerprint = prepare_session.call_args.kwargs["operation_fingerprint"] + + assert first_fingerprint != second_fingerprint def test_prepare_put_execution_propagates_missing_source(tmp_path: Path) -> None: diff --git a/tests/application/publish/test_session.py b/tests/application/publish/test_session.py index 48ac02b6..2d55f832 100644 --- a/tests/application/publish/test_session.py +++ b/tests/application/publish/test_session.py @@ -5,7 +5,11 @@ import pytest -from roar.application.publish.session import PreparedPublishSession, prepare_publish_session +from roar.application.publish.session import ( + PreparedPublishSession, + delegated_client_session_id, + prepare_publish_session, +) from roar.core.interfaces.lineage import LineageData from roar.core.interfaces.registration import GitContext, SessionRegistrationResult @@ -31,6 +35,32 @@ def _lineage() -> LineageData: ) +def test_delegated_client_session_id_is_operation_scoped(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ROAR_DELEGATED_JOB_ID", "job-1") + monkeypatch.setenv("ROAR_DELEGATED_EXECUTION_ATTEMPT_ID", "attempt-1") + monkeypatch.setenv("ROAR_DELEGATED_TASK_ID", "task-1") + + register_id = delegated_client_session_id( + operation_kind="register", + operation_fingerprint="lineage-a", + ) + + assert register_id == delegated_client_session_id( + operation_kind="register", + operation_fingerprint="lineage-a", + ) + assert register_id != delegated_client_session_id( + operation_kind="register", + operation_fingerprint="lineage-b", + ) + assert register_id != delegated_client_session_id( + operation_kind="put", + operation_fingerprint="lineage-a", + ) + assert register_id is not None + assert register_id.startswith("roar-delegated-v2-") + + def test_prepare_publish_session_computes_hash_without_registering(tmp_path: Path) -> None: glaas_client = MagicMock() session_service = MagicMock() From abf56d6a450cfb5fa24b690fe5eb43f92d8f4f06 Mon Sep 17 00:00:00 2001 From: Trevor Basinger Date: Tue, 18 Aug 2026 16:51:53 +0000 Subject: [PATCH 26/52] style(publish): use modern mapping import --- roar/application/publish/put_preparation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roar/application/publish/put_preparation.py b/roar/application/publish/put_preparation.py index aa43bba9..8c9cf97c 100644 --- a/roar/application/publish/put_preparation.py +++ b/roar/application/publish/put_preparation.py @@ -5,9 +5,10 @@ import hashlib import json import os +from collections.abc import Mapping from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, Any, Mapping +from typing import TYPE_CHECKING, Any from urllib.parse import urlparse from ...core.interfaces.logger import ILogger From e0a27de662f3610502cb4ed91e0b00a837ef5361 Mon Sep 17 00:00:00 2001 From: Trevor Basinger Date: Tue, 18 Aug 2026 17:15:05 +0000 Subject: [PATCH 27/52] fix(publish): persist delegated put identity --- roar/application/publish/put_preparation.py | 217 +++++++++++++++++- roar/application/publish/service.py | 10 + roar/db/schema.py | 36 ++- .../publish/test_put_preparation.py | 160 ++++++++++++- tests/application/publish/test_service.py | 4 + 5 files changed, 417 insertions(+), 10 deletions(-) diff --git a/roar/application/publish/put_preparation.py b/roar/application/publish/put_preparation.py index 8c9cf97c..e02cee94 100644 --- a/roar/application/publish/put_preparation.py +++ b/roar/application/publish/put_preparation.py @@ -5,12 +5,15 @@ import hashlib import json import os +import time from collections.abc import Mapping from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Any from urllib.parse import urlparse +from sqlalchemy import text + from ...core.interfaces.logger import ILogger from ...core.interfaces.registration import GitContext from ...db.hashing import hash_files_blake3 @@ -28,6 +31,16 @@ from .source_resolution import ResolvedSource +@dataclass(frozen=True) +class DelegatedPutOperation: + """Durable local reservation for one broker-backed put operation.""" + + task_identity: str + session_id: int + ordinal: int + request_fingerprint: str + + @dataclass(frozen=True) class PreparedPutExecution: """Application-prepared context for a put execution.""" @@ -46,6 +59,7 @@ class PreparedPutExecution: registration_session_status: str | None = None dataset_identifiers: list[dict[str, Any]] = field(default_factory=list) additional_composite_roots: dict[Path, list[ResolvedSource]] = field(default_factory=dict) + delegated_put_operation: DelegatedPutOperation | None = None def prepare_put_execution( @@ -96,8 +110,13 @@ def prepare_put_execution( if missing_hashes: raise OSError(f"Failed to hash put source: {missing_hashes[0]}") - operation_payload = { + operation_payload: dict[str, Any] = { "destination": destination, + "git": { + "branch": git_context.branch, + "commit": git_context.commit, + "repo": git_context.repo, + }, "local_session_hash": runtime.session_service.compute_session_hash( roar_dir=str(roar_dir), session_id=session_id, @@ -117,13 +136,26 @@ def prepare_put_execution( key=lambda source: (source["path"], source["relative_key"]), ), } - operation_fingerprint = hashlib.sha256( - json.dumps( - operation_payload, - sort_keys=True, - separators=(",", ":"), - ).encode() - ).hexdigest() + delegated_task_identity = _delegated_task_identity() + if delegated_task_identity is not None: + operation_payload["lineage_revision"] = _local_lineage_revision(db_ctx, session_id) + request_fingerprint = _fingerprint(operation_payload) + delegated_put_operation = _reserve_delegated_put_operation( + db_ctx=db_ctx, + delegated_task_identity=delegated_task_identity, + session_id=session_id, + request_fingerprint=request_fingerprint, + ) + operation_fingerprint = ( + _fingerprint( + { + "ordinal": delegated_put_operation.ordinal, + "request_fingerprint": request_fingerprint, + } + ) + if delegated_put_operation is not None + else request_fingerprint + ) publish_session = prepare_publish_session( remote_registry=remote_registry, @@ -168,7 +200,176 @@ def prepare_put_execution( source_hashes=source_hashes, dataset_identifiers=dataset_identifiers, additional_composite_roots=additional_composite_roots, + delegated_put_operation=delegated_put_operation, + ) + + +def complete_delegated_put_operation( + db_ctx: Any, + operation: DelegatedPutOperation | None, +) -> None: + """Mark a broker-backed put complete after its local and remote writes succeed.""" + if not isinstance(operation, DelegatedPutOperation): + return + + completed_at = time.time() + result = db_ctx.session.execute( + text( + """ + UPDATE delegated_put_operations + SET status = 'completed', updated_at = :completed_at, completed_at = :completed_at + WHERE task_identity = :task_identity + AND session_id = :session_id + AND ordinal = :ordinal + AND request_fingerprint = :request_fingerprint + AND status = 'pending' + """ + ), + { + "completed_at": completed_at, + "task_identity": operation.task_identity, + "session_id": operation.session_id, + "ordinal": operation.ordinal, + "request_fingerprint": operation.request_fingerprint, + }, ) + if result.rowcount != 1: + raise RuntimeError("Delegated put operation reservation changed before completion") + db_ctx.commit() + + +def _delegated_task_identity() -> str | None: + values = [ + os.environ.get("ROAR_DELEGATED_JOB_ID", "").strip(), + os.environ.get("ROAR_DELEGATED_EXECUTION_ATTEMPT_ID", "").strip(), + os.environ.get("ROAR_DELEGATED_TASK_ID", "").strip(), + ] + if not any(values): + return None + if not all(values): + raise ValueError("Delegated publication task identity is incomplete") + return hashlib.sha256("\0".join(values).encode()).hexdigest() + + +def _local_lineage_revision(db_ctx: Any, session_id: int) -> str: + jobs = db_ctx.session.execute( + text( + """ + SELECT id, job_uid, step_number, step_identity, command, git_repo, + git_commit, git_branch, status, exit_code + FROM jobs + WHERE session_id = :session_id + ORDER BY id + """ + ), + {"session_id": session_id}, + ).mappings() + links = db_ctx.session.execute( + text( + """ + SELECT 'input' AS relation, link.job_id, link.artifact_id, link.path + FROM job_inputs AS link + JOIN jobs AS job ON job.id = link.job_id + WHERE job.session_id = :session_id + UNION ALL + SELECT 'output' AS relation, link.job_id, link.artifact_id, link.path + FROM job_outputs AS link + JOIN jobs AS job ON job.id = link.job_id + WHERE job.session_id = :session_id + ORDER BY relation, job_id, artifact_id, path + """ + ), + {"session_id": session_id}, + ).mappings() + return _fingerprint( + { + "jobs": [dict(row) for row in jobs], + "links": [dict(row) for row in links], + } + ) + + +def _reserve_delegated_put_operation( + *, + db_ctx: Any, + delegated_task_identity: str | None, + session_id: int, + request_fingerprint: str, +) -> DelegatedPutOperation | None: + if delegated_task_identity is None: + return None + + now = time.time() + row = ( + db_ctx.session.execute( + text( + """ + INSERT INTO delegated_put_operations ( + task_identity, + session_id, + ordinal, + request_fingerprint, + status, + created_at, + updated_at, + completed_at + ) VALUES ( + :task_identity, + :session_id, + 1, + :request_fingerprint, + 'pending', + :now, + :now, + NULL + ) + ON CONFLICT(task_identity, session_id) DO UPDATE SET + ordinal = CASE + WHEN delegated_put_operations.status = 'completed' + THEN delegated_put_operations.ordinal + 1 + ELSE delegated_put_operations.ordinal + END, + request_fingerprint = CASE + WHEN delegated_put_operations.status = 'completed' + THEN excluded.request_fingerprint + ELSE delegated_put_operations.request_fingerprint + END, + status = 'pending', + updated_at = excluded.updated_at, + completed_at = NULL + WHERE delegated_put_operations.status = 'completed' + OR delegated_put_operations.request_fingerprint = excluded.request_fingerprint + RETURNING ordinal, request_fingerprint + """ + ), + { + "task_identity": delegated_task_identity, + "session_id": session_id, + "request_fingerprint": request_fingerprint, + "now": now, + }, + ) + .mappings() + .one_or_none() + ) + if row is None: + raise ValueError( + "A different delegated put operation is already pending for this task; " + "retry the original command" + ) + db_ctx.commit() + return DelegatedPutOperation( + task_identity=delegated_task_identity, + session_id=session_id, + ordinal=int(row["ordinal"]), + request_fingerprint=str(row["request_fingerprint"]), + ) + + +def _fingerprint(payload: Mapping[str, Any]) -> str: + return hashlib.sha256( + json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() def _destination_type(destination: str) -> str: diff --git a/roar/application/publish/service.py b/roar/application/publish/service.py index 68e9a90e..8fe9264e 100644 --- a/roar/application/publish/service.py +++ b/roar/application/publish/service.py @@ -99,6 +99,13 @@ def prepare_put_execution(*args: Any, **kwargs: Any) -> Any: return _prepare_put_execution(*args, **kwargs) +def complete_delegated_put_operation(*args: Any, **kwargs: Any) -> Any: + """Durably close a delegated put reservation after publication succeeds.""" + from .put_preparation import complete_delegated_put_operation as _complete + + return _complete(*args, **kwargs) + + def prepare_register_execution(*args: Any, **kwargs: Any) -> Any: """Load register preparation only when register runs.""" from .register_preparation import ( @@ -930,6 +937,9 @@ def put_artifacts(request: PutRequest) -> PutResponse: ), ) + if result.success: + complete_delegated_put_operation(db_ctx, prepared.delegated_put_operation) + # Apply step name label if provided. if request.step_name and result.success and result.job_id: with contextlib.suppress(Exception): diff --git a/roar/db/schema.py b/roar/db/schema.py index 7eefd3c0..b4c609f3 100644 --- a/roar/db/schema.py +++ b/roar/db/schema.py @@ -245,10 +245,28 @@ CREATE INDEX IF NOT EXISTS idx_hash_cache_path ON hash_cache(path); CREATE INDEX IF NOT EXISTS idx_hash_cache_updated ON hash_cache(cached_at); + +-- ============================================================================= +-- DELEGATED PUT OPERATIONS +-- Durable retry identity for one broker-backed put operation. A pending row is +-- reused after an interrupted response; completing it advances the ordinal for +-- the next command, even when the request is otherwise identical. +-- ============================================================================= +CREATE TABLE IF NOT EXISTS delegated_put_operations ( + task_identity TEXT NOT NULL, + session_id INTEGER NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + ordinal INTEGER NOT NULL, + request_fingerprint TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'completed')), + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + completed_at REAL, + PRIMARY KEY (task_identity, session_id) +); """ -_SCHEMA_VERSION = 3 # Bump when adding new migrations below. +_SCHEMA_VERSION = 4 # Bump when adding new migrations below. def run_migrations(conn) -> None: @@ -365,5 +383,21 @@ def run_migrations(conn) -> None: if "write_origin" not in label_columns: conn.execute("ALTER TABLE labels ADD COLUMN write_origin TEXT") + conn.execute( + """ + CREATE TABLE IF NOT EXISTS delegated_put_operations ( + task_identity TEXT NOT NULL, + session_id INTEGER NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + ordinal INTEGER NOT NULL, + request_fingerprint TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'completed')), + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + completed_at REAL, + PRIMARY KEY (task_identity, session_id) + ) + """ + ) + # Stamp the schema version so subsequent opens skip the full migration check. conn.execute(f"PRAGMA user_version = {_SCHEMA_VERSION}") diff --git a/tests/application/publish/test_put_preparation.py b/tests/application/publish/test_put_preparation.py index 96d598b5..c8379457 100644 --- a/tests/application/publish/test_put_preparation.py +++ b/tests/application/publish/test_put_preparation.py @@ -4,9 +4,15 @@ from unittest.mock import MagicMock, patch import pytest +from sqlalchemy import text -from roar.application.publish.put_preparation import PreparedPutExecution, prepare_put_execution +from roar.application.publish.put_preparation import ( + PreparedPutExecution, + complete_delegated_put_operation, + prepare_put_execution, +) from roar.core.interfaces.registration import GitContext +from roar.db.context import create_database_context def test_prepare_put_execution_requires_active_session(tmp_path: Path) -> None: @@ -174,3 +180,155 @@ def test_prepare_put_execution_propagates_missing_source(tmp_path: Path) -> None git_commit="deadbeef", logger=MagicMock(), ) + + +def test_delegated_put_reuses_pending_retry_then_advances_identical_operation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _set_delegated_task(monkeypatch) + model = tmp_path / "model.pt" + model.write_bytes(b"model") + roar_dir = tmp_path / ".roar" + runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" + git_context = GitContext(repo="repo", branch="main", commit="deadbeef") + + with create_database_context(roar_dir) as db_ctx: + db_ctx.sessions.get_or_create_active() + db_ctx.commit() + with ( + patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=git_context, + ), + patch( + "roar.application.publish.put_preparation.prepare_publish_session", + return_value=MagicMock( + session_hash="session-hash", + session_url=None, + registration_session_id="registration-session", + registration_session_mode="delegated", + registration_session_status="active", + ), + ) as prepare_session, + ): + first = _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + first_fingerprint = prepare_session.call_args.kwargs["operation_fingerprint"] + retry = _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + retry_fingerprint = prepare_session.call_args.kwargs["operation_fingerprint"] + + assert first.delegated_put_operation is not None + assert retry.delegated_put_operation == first.delegated_put_operation + assert retry_fingerprint == first_fingerprint + + complete_delegated_put_operation(db_ctx, retry.delegated_put_operation) + second = _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + second_fingerprint = prepare_session.call_args.kwargs["operation_fingerprint"] + + assert second.delegated_put_operation is not None + assert second.delegated_put_operation.ordinal == 2 + assert second_fingerprint != first_fingerprint + + +def test_delegated_put_rejects_changed_git_context_while_retry_is_pending( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _set_delegated_task(monkeypatch) + (tmp_path / "model.pt").write_bytes(b"model") + roar_dir = tmp_path / ".roar" + runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" + + with create_database_context(roar_dir) as db_ctx: + db_ctx.sessions.get_or_create_active() + db_ctx.commit() + with patch( + "roar.application.publish.put_preparation.prepare_publish_session", + return_value=MagicMock( + session_hash="session-hash", + session_url=None, + registration_session_id="registration-session", + registration_session_mode="delegated", + registration_session_status="active", + ), + ): + with patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=GitContext(repo="repo", branch="main", commit="first"), + ): + _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + with ( + patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=GitContext(repo="repo", branch="feature", commit="second"), + ), + pytest.raises(ValueError, match="different delegated put operation"), + ): + _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + + +def test_delegated_put_rejects_changed_lineage_while_retry_is_pending( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _set_delegated_task(monkeypatch) + (tmp_path / "model.pt").write_bytes(b"model") + roar_dir = tmp_path / ".roar" + runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" + git_context = GitContext(repo="repo", branch="main", commit="deadbeef") + + with create_database_context(roar_dir) as db_ctx: + session_id = db_ctx.sessions.get_or_create_active() + db_ctx.commit() + with ( + patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=git_context, + ), + patch( + "roar.application.publish.put_preparation.prepare_publish_session", + return_value=MagicMock( + session_hash="session-hash", + session_url=None, + registration_session_id="registration-session", + registration_session_mode="delegated", + registration_session_status="active", + ), + ), + ): + _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + db_ctx.session.execute( + text( + """ + INSERT INTO jobs (timestamp, command, session_id, step_number) + VALUES (1, 'python upstream.py', :session_id, 1) + """ + ), + {"session_id": session_id}, + ) + db_ctx.commit() + + with pytest.raises(ValueError, match="different delegated put operation"): + _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + + +def _set_delegated_task(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ROAR_DELEGATED_JOB_ID", "job-1") + monkeypatch.setenv("ROAR_DELEGATED_EXECUTION_ATTEMPT_ID", "attempt-1") + monkeypatch.setenv("ROAR_DELEGATED_TASK_ID", "task-1") + + +def _prepare_model_put(db_ctx, runtime, roar_dir: Path, repo_root: Path) -> PreparedPutExecution: + return prepare_put_execution( + db_ctx=db_ctx, + runtime=runtime, + roar_dir=roar_dir, + repo_root=repo_root, + sources=["model.pt"], + destination="memory://bucket/prefix", + git_commit="deadbeef", + logger=MagicMock(), + ) diff --git a/tests/application/publish/test_service.py b/tests/application/publish/test_service.py index 55809746..8fe2c7a9 100644 --- a/tests/application/publish/test_service.py +++ b/tests/application/publish/test_service.py @@ -390,6 +390,9 @@ def test_put_artifacts_continues_when_git_preflight_warns(tmp_path: Path) -> Non "roar.application.publish.service.finalize_put_git", return_value=(None, []), ), + patch( + "roar.application.publish.service.complete_delegated_put_operation" + ) as complete_operation, ): mock_put_cls.return_value.put_prepared.return_value = put_result @@ -414,6 +417,7 @@ def test_put_artifacts_continues_when_git_preflight_warns(tmp_path: Path) -> Non reproducible=False, commit_on_remote=False, ) + complete_operation.assert_called_once_with(db_ctx, prepared.delegated_put_operation) def test_put_artifacts_returns_preparation_error_before_service(tmp_path: Path) -> None: From 6f533c2cc13cae3dffb2f7fd734bfb5590a6b7cc Mon Sep 17 00:00:00 2001 From: Trevor Basinger Date: Tue, 18 Aug 2026 18:03:20 +0000 Subject: [PATCH 28/52] fix(publish): make delegated put retries durable --- roar/application/publish/put_execution.py | 38 ++-- roar/application/publish/put_preparation.py | 183 +++++++++++++++++- roar/db/schema.py | 21 +- .../publish/test_put_preparation.py | 179 +++++++++++++++++ tests/unit/put/test_put_service.py | 88 ++++++++- tests/unit/test_schema_parent_job_uid.py | 33 ++++ 6 files changed, 519 insertions(+), 23 deletions(-) diff --git a/roar/application/publish/put_execution.py b/roar/application/publish/put_execution.py index dfe01e5c..aca63cde 100644 --- a/roar/application/publish/put_execution.py +++ b/roar/application/publish/put_execution.py @@ -19,7 +19,7 @@ from ...application.publish.composites import build_publish_composite_results from ...application.publish.lineage import LineageCollector from ...application.publish.metadata import build_put_operation_metadata_json -from ...application.publish.put_preparation import PreparedPutExecution +from ...application.publish.put_preparation import DelegatedPutOperation, PreparedPutExecution from ...application.publish.registration import ( normalize_registration_hashes, normalize_registration_source_type, @@ -360,6 +360,7 @@ def put_prepared( coordinator=coordinator, registration_session_id=registration_session_id, registration_session_mode=registration_session_mode, + delegated_put_operation=prepared.delegated_put_operation, session_id=session_id, fallback_session_hash=session_hash or "", git_context=git_context, @@ -620,6 +621,7 @@ def _put_prepared_with_registration_session( coordinator: RegistrationCoordinator, registration_session_id: str, registration_session_mode: str | None, + delegated_put_operation: DelegatedPutOperation | None, session_id: int, fallback_session_hash: str, git_context: GitContext, @@ -663,18 +665,30 @@ def _put_prepared_with_registration_session( timestamp=time.time(), ) - step_number = self._db.sessions.get_next_step_number(session_id) - job_id, job_uid = self._db.jobs.create( - command=command, - timestamp=time.time(), - session_id=session_id, - step_number=step_number, - metadata=provisional_metadata_json, - execution_backend="local", - execution_role="host", - job_type="put", - exit_code=0, + stable_put_job_uid = ( + delegated_put_operation.put_job_uid if delegated_put_operation is not None else None ) + existing_put_job = ( + self._db.jobs.get_by_uid(stable_put_job_uid) if stable_put_job_uid else None + ) + if existing_put_job is not None: + job_id = int(existing_put_job["id"]) + job_uid = str(existing_put_job["job_uid"]) + step_number = int(existing_put_job.get("step_number") or 0) + else: + step_number = self._db.sessions.get_next_step_number(session_id) + job_id, job_uid = self._db.jobs.create( + command=command, + timestamp=time.time(), + job_uid=stable_put_job_uid, + session_id=session_id, + step_number=step_number, + metadata=provisional_metadata_json, + execution_backend="local", + execution_role="host", + job_type="put", + exit_code=0, + ) self._logger.debug( "Put job created before registration-session finalize: id=%s, uid=%s, step=%d", job_id, diff --git a/roar/application/publish/put_preparation.py b/roar/application/publish/put_preparation.py index e02cee94..68eceda5 100644 --- a/roar/application/publish/put_preparation.py +++ b/roar/application/publish/put_preparation.py @@ -5,6 +5,7 @@ import hashlib import json import os +import secrets import time from collections.abc import Mapping from dataclasses import dataclass, field @@ -39,6 +40,7 @@ class DelegatedPutOperation: session_id: int ordinal: int request_fingerprint: str + put_job_uid: str @dataclass(frozen=True) @@ -138,7 +140,16 @@ def prepare_put_execution( } delegated_task_identity = _delegated_task_identity() if delegated_task_identity is not None: - operation_payload["lineage_revision"] = _local_lineage_revision(db_ctx, session_id) + pending_put_job_uid = _pending_put_job_uid( + db_ctx, + delegated_task_identity, + session_id, + ) + operation_payload["lineage_revision"] = _local_lineage_revision( + db_ctx, + session_id, + exclude_job_uid=pending_put_job_uid, + ) request_fingerprint = _fingerprint(operation_payload) delegated_put_operation = _reserve_delegated_put_operation( db_ctx=db_ctx, @@ -251,40 +262,184 @@ def _delegated_task_identity() -> str | None: return hashlib.sha256("\0".join(values).encode()).hexdigest() -def _local_lineage_revision(db_ctx: Any, session_id: int) -> str: +def _pending_put_job_uid(db_ctx: Any, task_identity: str, session_id: int) -> str | None: + row = ( + db_ctx.session.execute( + text( + """ + SELECT put_job_uid + FROM delegated_put_operations + WHERE task_identity = :task_identity + AND session_id = :session_id + AND status = 'pending' + """ + ), + {"task_identity": task_identity, "session_id": session_id}, + ) + .mappings() + .one_or_none() + ) + return str(row["put_job_uid"]) if row and row["put_job_uid"] else None + + +def _local_lineage_revision( + db_ctx: Any, + session_id: int, + *, + exclude_job_uid: str | None = None, +) -> str: + query_params = { + "session_id": session_id, + "exclude_job_uid": exclude_job_uid, + } jobs = db_ctx.session.execute( text( """ - SELECT id, job_uid, step_number, step_identity, command, git_repo, - git_commit, git_branch, status, exit_code + SELECT id, job_uid, parent_job_uid, timestamp, command, + step_number, step_identity, git_repo, git_commit, git_branch, + duration_seconds, exit_code, status, execution_backend, + execution_role, job_type, metadata FROM jobs WHERE session_id = :session_id + AND (:exclude_job_uid IS NULL OR job_uid IS NULL OR job_uid != :exclude_job_uid) ORDER BY id """ ), - {"session_id": session_id}, + query_params, ).mappings() links = db_ctx.session.execute( text( """ - SELECT 'input' AS relation, link.job_id, link.artifact_id, link.path + SELECT 'input' AS relation, link.job_id, link.artifact_id, link.path, + link.byte_ranges FROM job_inputs AS link JOIN jobs AS job ON job.id = link.job_id WHERE job.session_id = :session_id + AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) UNION ALL - SELECT 'output' AS relation, link.job_id, link.artifact_id, link.path + SELECT 'output' AS relation, link.job_id, link.artifact_id, link.path, + link.byte_ranges FROM job_outputs AS link JOIN jobs AS job ON job.id = link.job_id WHERE job.session_id = :session_id + AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) ORDER BY relation, job_id, artifact_id, path """ ), - {"session_id": session_id}, + query_params, + ).mappings() + artifacts = db_ctx.session.execute( + text( + """ + WITH lineage_artifacts AS ( + SELECT link.artifact_id + FROM job_inputs AS link + JOIN jobs AS job ON job.id = link.job_id + WHERE job.session_id = :session_id + AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) + UNION + SELECT link.artifact_id + FROM job_outputs AS link + JOIN jobs AS job ON job.id = link.job_id + WHERE job.session_id = :session_id + AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) + ) + SELECT artifact.id, artifact.size, artifact.first_seen_path, + artifact.source_type, artifact.source_url, artifact.capture_method, + artifact.kind, artifact.component_count, artifact.metadata + FROM artifacts AS artifact + JOIN lineage_artifacts ON lineage_artifacts.artifact_id = artifact.id + ORDER BY artifact.id + """ + ), + query_params, + ).mappings() + artifact_hashes = db_ctx.session.execute( + text( + """ + WITH lineage_artifacts AS ( + SELECT link.artifact_id + FROM job_inputs AS link + JOIN jobs AS job ON job.id = link.job_id + WHERE job.session_id = :session_id + AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) + UNION + SELECT link.artifact_id + FROM job_outputs AS link + JOIN jobs AS job ON job.id = link.job_id + WHERE job.session_id = :session_id + AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) + ) + SELECT hashes.artifact_id, hashes.algorithm, hashes.digest + FROM artifact_hashes AS hashes + JOIN lineage_artifacts ON lineage_artifacts.artifact_id = hashes.artifact_id + ORDER BY hashes.artifact_id, hashes.algorithm, hashes.digest + """ + ), + query_params, + ).mappings() + composite_components = db_ctx.session.execute( + text( + """ + WITH lineage_artifacts AS ( + SELECT link.artifact_id + FROM job_inputs AS link + JOIN jobs AS job ON job.id = link.job_id + WHERE job.session_id = :session_id + AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) + UNION + SELECT link.artifact_id + FROM job_outputs AS link + JOIN jobs AS job ON job.id = link.job_id + WHERE job.session_id = :session_id + AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) + ) + SELECT component.composite_artifact_id, component.ordinal, + component.relative_path, component.leaf_kind, + component.component_algorithm, component.component_digest, + component.component_size, component.component_type + FROM composite_artifact_components AS component + JOIN lineage_artifacts + ON lineage_artifacts.artifact_id = component.composite_artifact_id + ORDER BY component.composite_artifact_id, component.ordinal, + component.relative_path + """ + ), + query_params, + ).mappings() + membership_indexes = db_ctx.session.execute( + text( + """ + WITH lineage_artifacts AS ( + SELECT link.artifact_id + FROM job_inputs AS link + JOIN jobs AS job ON job.id = link.job_id + WHERE job.session_id = :session_id + AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) + UNION + SELECT link.artifact_id + FROM job_outputs AS link + JOIN jobs AS job ON job.id = link.job_id + WHERE job.session_id = :session_id + AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) + ) + SELECT membership.* + FROM composite_membership_indexes AS membership + JOIN lineage_artifacts + ON lineage_artifacts.artifact_id = membership.composite_artifact_id + ORDER BY membership.composite_artifact_id + """ + ), + query_params, ).mappings() return _fingerprint( { "jobs": [dict(row) for row in jobs], "links": [dict(row) for row in links], + "artifacts": [dict(row) for row in artifacts], + "artifact_hashes": [dict(row) for row in artifact_hashes], + "composite_components": [dict(row) for row in composite_components], + "membership_indexes": [dict(row) for row in membership_indexes], } ) @@ -300,6 +455,7 @@ def _reserve_delegated_put_operation( return None now = time.time() + candidate_put_job_uid = f"delegated-put-{secrets.token_hex(12)}" row = ( db_ctx.session.execute( text( @@ -309,6 +465,7 @@ def _reserve_delegated_put_operation( session_id, ordinal, request_fingerprint, + put_job_uid, status, created_at, updated_at, @@ -318,6 +475,7 @@ def _reserve_delegated_put_operation( :session_id, 1, :request_fingerprint, + :put_job_uid, 'pending', :now, :now, @@ -334,18 +492,24 @@ def _reserve_delegated_put_operation( THEN excluded.request_fingerprint ELSE delegated_put_operations.request_fingerprint END, + put_job_uid = CASE + WHEN delegated_put_operations.status = 'completed' + THEN excluded.put_job_uid + ELSE delegated_put_operations.put_job_uid + END, status = 'pending', updated_at = excluded.updated_at, completed_at = NULL WHERE delegated_put_operations.status = 'completed' OR delegated_put_operations.request_fingerprint = excluded.request_fingerprint - RETURNING ordinal, request_fingerprint + RETURNING ordinal, request_fingerprint, put_job_uid """ ), { "task_identity": delegated_task_identity, "session_id": session_id, "request_fingerprint": request_fingerprint, + "put_job_uid": candidate_put_job_uid, "now": now, }, ) @@ -363,6 +527,7 @@ def _reserve_delegated_put_operation( session_id=session_id, ordinal=int(row["ordinal"]), request_fingerprint=str(row["request_fingerprint"]), + put_job_uid=str(row["put_job_uid"]), ) diff --git a/roar/db/schema.py b/roar/db/schema.py index b4c609f3..b97eb05b 100644 --- a/roar/db/schema.py +++ b/roar/db/schema.py @@ -257,6 +257,7 @@ session_id INTEGER NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, ordinal INTEGER NOT NULL, request_fingerprint TEXT NOT NULL, + put_job_uid TEXT NOT NULL, status TEXT NOT NULL CHECK (status IN ('pending', 'completed')), created_at REAL NOT NULL, updated_at REAL NOT NULL, @@ -266,7 +267,7 @@ """ -_SCHEMA_VERSION = 4 # Bump when adding new migrations below. +_SCHEMA_VERSION = 5 # Bump when adding new migrations below. def run_migrations(conn) -> None: @@ -390,6 +391,7 @@ def run_migrations(conn) -> None: session_id INTEGER NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, ordinal INTEGER NOT NULL, request_fingerprint TEXT NOT NULL, + put_job_uid TEXT NOT NULL, status TEXT NOT NULL CHECK (status IN ('pending', 'completed')), created_at REAL NOT NULL, updated_at REAL NOT NULL, @@ -399,5 +401,22 @@ def run_migrations(conn) -> None: """ ) + delegated_put_columns = { + row["name"] + for row in conn.execute("PRAGMA table_info(delegated_put_operations)").fetchall() + } + if "put_job_uid" not in delegated_put_columns: + conn.execute( + "ALTER TABLE delegated_put_operations ADD COLUMN put_job_uid TEXT NOT NULL DEFAULT ''" + ) + conn.execute( + """ + UPDATE delegated_put_operations + SET put_job_uid = 'delegated-put-' || substr(task_identity, 1, 16) + || '-' || session_id || '-' || ordinal + WHERE put_job_uid = '' + """ + ) + # Stamp the schema version so subsequent opens skip the full migration check. conn.execute(f"PRAGMA user_version = {_SCHEMA_VERSION}") diff --git a/tests/application/publish/test_put_preparation.py b/tests/application/publish/test_put_preparation.py index c8379457..08acfe0a 100644 --- a/tests/application/publish/test_put_preparation.py +++ b/tests/application/publish/test_put_preparation.py @@ -6,6 +6,7 @@ import pytest from sqlalchemy import text +from roar.application.publish.put_execution import PutService from roar.application.publish.put_preparation import ( PreparedPutExecution, complete_delegated_put_operation, @@ -13,6 +14,7 @@ ) from roar.core.interfaces.registration import GitContext from roar.db.context import create_database_context +from roar.integrations.storage import MemoryBackend def test_prepare_put_execution_requires_active_session(tmp_path: Path) -> None: @@ -315,6 +317,183 @@ def test_delegated_put_rejects_changed_lineage_while_retry_is_pending( _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) +def test_delegated_put_retry_excludes_its_persisted_sink_and_recovers_closed_receipt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _set_delegated_task(monkeypatch) + (tmp_path / "model.pt").write_bytes(b"model") + roar_dir = tmp_path / ".roar" + runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" + git_context = GitContext(repo="repo", branch="main", commit="deadbeef") + + with create_database_context(roar_dir) as db_ctx: + session_id = db_ctx.sessions.get_or_create_active() + db_ctx.commit() + with ( + patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=git_context, + ), + patch( + "roar.application.publish.put_preparation.prepare_publish_session", + side_effect=[ + MagicMock( + session_hash="provisional-hash", + session_url=None, + registration_session_id="registration-session", + registration_session_mode="delegated", + registration_session_status="active", + ), + MagicMock( + session_hash="authoritative-hash", + session_url="https://glaas.example/dag/authoritative-hash", + registration_session_id="registration-session", + registration_session_mode="delegated", + registration_session_status="closed", + ), + ], + ), + ): + first = _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + assert first.delegated_put_operation is not None + put_job_uid = first.delegated_put_operation.put_job_uid + put_job_id, created_uid = db_ctx.jobs.create( + command="roar put model.pt memory://bucket/prefix", + timestamp=1, + job_uid=put_job_uid, + session_id=session_id, + step_number=1, + job_type="put", + ) + assert created_uid == put_job_uid + db_ctx.session.execute( + text( + """ + INSERT INTO artifacts (id, size, first_seen_at, first_seen_path) + VALUES ('put-artifact', 5, 1, 'model.pt') + """ + ) + ) + db_ctx.jobs.add_input(put_job_id, "put-artifact", "model.pt") + db_ctx.commit() + + retry = _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + assert retry.delegated_put_operation == first.delegated_put_operation + service = PutService( + db_context=db_ctx, + backend=MemoryBackend(bucket="bucket", prefix="prefix"), + destination="memory://bucket/prefix", + repo_root=tmp_path, + ) + result = service.put_prepared( + prepared=retry, + sources=["model.pt"], + message="retry", + ) + assert result.success is True + assert result.session_hash == "authoritative-hash" + complete_delegated_put_operation(db_ctx, retry.delegated_put_operation) + + row = ( + db_ctx.session.execute( + text( + """ + SELECT status, ordinal, put_job_uid + FROM delegated_put_operations + WHERE task_identity = :task_identity AND session_id = :session_id + """ + ), + { + "task_identity": retry.delegated_put_operation.task_identity, + "session_id": session_id, + }, + ) + .mappings() + .one() + ) + put_job_count = db_ctx.session.execute( + text("SELECT COUNT(*) FROM jobs WHERE job_uid = :job_uid"), + {"job_uid": put_job_uid}, + ).scalar_one() + + assert dict(row) == {"status": "completed", "ordinal": 1, "put_job_uid": put_job_uid} + assert put_job_count == 1 + + +@pytest.mark.parametrize( + ("mutation_sql", "params"), + [ + ("UPDATE jobs SET job_type = 'ray_task' WHERE job_uid = 'upstream'", {}), + ("UPDATE jobs SET parent_job_uid = 'parent' WHERE job_uid = 'upstream'", {}), + ( + "UPDATE jobs SET metadata = :metadata WHERE job_uid = 'upstream'", + {"metadata": '{"changed":true}'}, + ), + ( + "UPDATE job_inputs SET byte_ranges = '[[0,4]]' WHERE job_id = :job_id", + {"job_id": 1}, + ), + ], +) +def test_delegated_put_rejects_emitted_lineage_contract_changes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + mutation_sql: str, + params: dict[str, object], +) -> None: + _set_delegated_task(monkeypatch) + (tmp_path / "model.pt").write_bytes(b"model") + roar_dir = tmp_path / ".roar" + runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" + git_context = GitContext(repo="repo", branch="main", commit="deadbeef") + + with create_database_context(roar_dir) as db_ctx: + session_id = db_ctx.sessions.get_or_create_active() + job_id, _ = db_ctx.jobs.create( + command="python upstream.py", + timestamp=1, + job_uid="upstream", + session_id=session_id, + step_number=1, + metadata="{}", + ) + db_ctx.session.execute( + text( + """ + INSERT INTO artifacts (id, size, first_seen_at, first_seen_path) + VALUES ('upstream-artifact', 5, 1, 'input.bin') + """ + ) + ) + db_ctx.jobs.add_input(job_id, "upstream-artifact", "input.bin") + db_ctx.commit() + with ( + patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=git_context, + ), + patch( + "roar.application.publish.put_preparation.prepare_publish_session", + return_value=MagicMock( + session_hash="session-hash", + session_url=None, + registration_session_id="registration-session", + registration_session_mode="delegated", + registration_session_status="active", + ), + ), + ): + _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + bound_params = {**params, "job_id": job_id} + db_ctx.session.execute(text(mutation_sql), bound_params) + db_ctx.commit() + with pytest.raises(ValueError, match="different delegated put operation"): + _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + + def _set_delegated_task(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("ROAR_DELEGATED_JOB_ID", "job-1") monkeypatch.setenv("ROAR_DELEGATED_EXECUTION_ATTEMPT_ID", "attempt-1") diff --git a/tests/unit/put/test_put_service.py b/tests/unit/put/test_put_service.py index c810a8f8..ba87df71 100644 --- a/tests/unit/put/test_put_service.py +++ b/tests/unit/put/test_put_service.py @@ -10,7 +10,7 @@ from roar.application.publish.composite_builder import CompositeArtifactBuilder from roar.application.publish.put_execution import PutService -from roar.application.publish.put_preparation import PreparedPutExecution +from roar.application.publish.put_preparation import DelegatedPutOperation, PreparedPutExecution from roar.application.publish.registration import build_lineage_membership_index_payload from roar.application.publish.results import PutDryRunItem from roar.application.publish.source_resolution import ResolvedSource @@ -85,6 +85,7 @@ def _prepared_put( composite_source_type: str | None = None, registration_session_id: str | None = None, registration_session_status: str | None = None, + delegated_put_operation: DelegatedPutOperation | None = None, ) -> PreparedPutExecution: resolved: list[ResolvedSource] = [] for source in sources: @@ -124,6 +125,7 @@ def _prepared_put( composite_source_type=composite_source_type, registration_session_id=registration_session_id, registration_session_status=registration_session_status, + delegated_put_operation=delegated_put_operation, ) @@ -200,6 +202,90 @@ def test_put_prepared_single_file_creates_job(self, tmp_path: Path) -> None: assert call_kwargs["job_type"] == "put" service._db.jobs.add_input.assert_called_once() + def test_active_delegated_retry_reuses_reserved_local_put_job(self, tmp_path: Path) -> None: + model_file = tmp_path / "model.pt" + model_file.write_bytes(b"model data") + db = _create_mock_db() + operation = DelegatedPutOperation( + task_identity="task", + session_id=1, + ordinal=1, + request_fingerprint="fingerprint", + put_job_uid="delegated-put-stable", + ) + db.jobs.get_by_uid.return_value = { + "id": 42, + "job_uid": operation.put_job_uid, + "step_number": 3, + } + coordinator = _create_mock_coordinator() + coordinator.register_lineage_under_registration_session.return_value = ( + BatchRegistrationResult( + session_registered=True, + jobs_created=0, + jobs_failed=0, + artifacts_registered=0, + artifacts_failed=0, + links_created=0, + links_failed=0, + errors=[], + ) + ) + coordinator.job_service.create_job_under_registration_session.return_value = ( + JobRegistrationResult( + success=True, + job_uid="remote-put", + job_id="remote-put", + error=None, + ) + ) + coordinator.job_service.link_job_artifacts_under_registration_session.return_value = ( + JobLinkResult( + success=True, + job_uid="remote-put", + inputs_linked=1, + outputs_linked=0, + error=None, + ) + ) + coordinator.session_service.finalize_registration_session.return_value = MagicMock( + success=True, + session_hash="final-hash", + session_url="https://glaas.example/dag/final-hash", + error=None, + ) + service = PutService( + db_context=db, + backend=MemoryBackend(bucket="test-bucket", prefix="models"), + destination="memory://test-bucket/models", + repo_root=tmp_path, + lineage_collector=MagicMock(), + registration_coordinator=coordinator, + ) + service._lineage_collector.collect.return_value = LineageData( + jobs=[], + artifacts=[], + artifact_hashes=set(), + pipeline={"id": 1}, + ) + + result = service.put_prepared( + prepared=_prepared_put( + tmp_path, + sources=[model_file], + registration_session_id="registration-session", + registration_session_status="active", + delegated_put_operation=operation, + ), + sources=[str(model_file)], + message="retry", + ) + + assert result.success is True + assert result.job_id == 42 + db.jobs.get_by_uid.assert_called_once_with(operation.put_job_uid) + db.jobs.create.assert_not_called() + def test_put_prepared_refreshes_and_syncs_put_job_labels(self, tmp_path: Path) -> None: model_file = tmp_path / "model.pt" model_file.write_bytes(b"model data") diff --git a/tests/unit/test_schema_parent_job_uid.py b/tests/unit/test_schema_parent_job_uid.py index 8ac03c7e..4e63592c 100644 --- a/tests/unit/test_schema_parent_job_uid.py +++ b/tests/unit/test_schema_parent_job_uid.py @@ -103,6 +103,39 @@ def test_insert_job_with_null_parent_job_uid_succeeds() -> None: assert row["parent_job_uid"] is None +def test_run_migrations_adds_stable_put_job_uid_to_v4_reservations() -> None: + conn = _create_legacy_db() + conn.executescript( + """ + CREATE TABLE sessions (id INTEGER PRIMARY KEY); + INSERT INTO sessions (id) VALUES (7); + CREATE TABLE delegated_put_operations ( + task_identity TEXT NOT NULL, + session_id INTEGER NOT NULL, + ordinal INTEGER NOT NULL, + request_fingerprint TEXT NOT NULL, + status TEXT NOT NULL, + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + completed_at REAL, + PRIMARY KEY (task_identity, session_id) + ); + INSERT INTO delegated_put_operations ( + task_identity, session_id, ordinal, request_fingerprint, + status, created_at, updated_at + ) VALUES ('abcdef0123456789', 7, 2, 'fingerprint', 'pending', 1, 1); + PRAGMA user_version = 4; + """ + ) + + run_migrations(conn) + + row = conn.execute("SELECT put_job_uid FROM delegated_put_operations").fetchone() + assert row is not None + assert row["put_job_uid"] == "delegated-put-abcdef0123456789-7-2" + assert conn.execute("PRAGMA user_version").fetchone()[0] == 5 + + def test_create_database_context_migrates_parent_job_uid_for_legacy_db(tmp_path: Path) -> None: roar_dir = tmp_path / ".roar" roar_dir.mkdir() From 71f8ff79b137b1fb1ce936ca92f28a19599daf39 Mon Sep 17 00:00:00 2001 From: Trevor Basinger Date: Tue, 18 Aug 2026 18:45:08 +0000 Subject: [PATCH 29/52] fix(publish): fingerprint the staged put lineage --- roar/application/publish/put_execution.py | 7 +- roar/application/publish/put_preparation.py | 212 +++++------------- .../publish/test_put_preparation.py | 154 +++++++++++-- tests/unit/put/test_put_service.py | 31 +++ 4 files changed, 229 insertions(+), 175 deletions(-) diff --git a/roar/application/publish/put_execution.py b/roar/application/publish/put_execution.py index aca63cde..907cc7dd 100644 --- a/roar/application/publish/put_execution.py +++ b/roar/application/publish/put_execution.py @@ -344,8 +344,11 @@ def put_prepared( ) # Collect lineage for all uploaded artifacts (merged) - collector = self._lineage_collector or LineageCollector() - lineage = collector.collect(artifact_hashes, self._roar_dir) + if prepared.lineage is not None: + lineage = prepared.lineage + else: + collector = self._lineage_collector or LineageCollector() + lineage = collector.collect(artifact_hashes, self._roar_dir) self._logger.debug( "Lineage collected: %d job(s), %d artifact(s)", len(lineage.jobs), diff --git a/roar/application/publish/put_preparation.py b/roar/application/publish/put_preparation.py index 68eceda5..7bd2d2bc 100644 --- a/roar/application/publish/put_preparation.py +++ b/roar/application/publish/put_preparation.py @@ -15,6 +15,7 @@ from sqlalchemy import text +from ...core.interfaces.lineage import LineageData from ...core.interfaces.logger import ILogger from ...core.interfaces.registration import GitContext from ...db.hashing import hash_files_blake3 @@ -62,6 +63,7 @@ class PreparedPutExecution: dataset_identifiers: list[dict[str, Any]] = field(default_factory=list) additional_composite_roots: dict[Path, list[ResolvedSource]] = field(default_factory=dict) delegated_put_operation: DelegatedPutOperation | None = None + lineage: LineageData | None = None def prepare_put_execution( @@ -139,16 +141,22 @@ def prepare_put_execution( ), } delegated_task_identity = _delegated_task_identity() + collected_lineage: LineageData | None = None if delegated_task_identity is not None: pending_put_job_uid = _pending_put_job_uid( db_ctx, delegated_task_identity, session_id, ) - operation_payload["lineage_revision"] = _local_lineage_revision( - db_ctx, - session_id, - exclude_job_uid=pending_put_job_uid, + from .lineage import LineageCollector + + collected_lineage = LineageCollector().collect( + sorted(set(source_hashes.values())), + roar_dir, + ) + collected_lineage = _without_lineage_job(collected_lineage, pending_put_job_uid) + operation_payload["lineage_revision"] = _collected_lineage_revision( + collected_lineage, ) request_fingerprint = _fingerprint(operation_payload) delegated_put_operation = _reserve_delegated_put_operation( @@ -212,6 +220,7 @@ def prepare_put_execution( dataset_identifiers=dataset_identifiers, additional_composite_roots=additional_composite_roots, delegated_put_operation=delegated_put_operation, + lineage=collected_lineage, ) @@ -282,168 +291,51 @@ def _pending_put_job_uid(db_ctx: Any, task_identity: str, session_id: int) -> st return str(row["put_job_uid"]) if row and row["put_job_uid"] else None -def _local_lineage_revision( - db_ctx: Any, - session_id: int, - *, - exclude_job_uid: str | None = None, -) -> str: - query_params = { - "session_id": session_id, - "exclude_job_uid": exclude_job_uid, - } - jobs = db_ctx.session.execute( - text( - """ - SELECT id, job_uid, parent_job_uid, timestamp, command, - step_number, step_identity, git_repo, git_commit, git_branch, - duration_seconds, exit_code, status, execution_backend, - execution_role, job_type, metadata - FROM jobs - WHERE session_id = :session_id - AND (:exclude_job_uid IS NULL OR job_uid IS NULL OR job_uid != :exclude_job_uid) - ORDER BY id - """ - ), - query_params, - ).mappings() - links = db_ctx.session.execute( - text( - """ - SELECT 'input' AS relation, link.job_id, link.artifact_id, link.path, - link.byte_ranges - FROM job_inputs AS link - JOIN jobs AS job ON job.id = link.job_id - WHERE job.session_id = :session_id - AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) - UNION ALL - SELECT 'output' AS relation, link.job_id, link.artifact_id, link.path, - link.byte_ranges - FROM job_outputs AS link - JOIN jobs AS job ON job.id = link.job_id - WHERE job.session_id = :session_id - AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) - ORDER BY relation, job_id, artifact_id, path - """ - ), - query_params, - ).mappings() - artifacts = db_ctx.session.execute( - text( - """ - WITH lineage_artifacts AS ( - SELECT link.artifact_id - FROM job_inputs AS link - JOIN jobs AS job ON job.id = link.job_id - WHERE job.session_id = :session_id - AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) - UNION - SELECT link.artifact_id - FROM job_outputs AS link - JOIN jobs AS job ON job.id = link.job_id - WHERE job.session_id = :session_id - AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) - ) - SELECT artifact.id, artifact.size, artifact.first_seen_path, - artifact.source_type, artifact.source_url, artifact.capture_method, - artifact.kind, artifact.component_count, artifact.metadata - FROM artifacts AS artifact - JOIN lineage_artifacts ON lineage_artifacts.artifact_id = artifact.id - ORDER BY artifact.id - """ - ), - query_params, - ).mappings() - artifact_hashes = db_ctx.session.execute( - text( - """ - WITH lineage_artifacts AS ( - SELECT link.artifact_id - FROM job_inputs AS link - JOIN jobs AS job ON job.id = link.job_id - WHERE job.session_id = :session_id - AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) - UNION - SELECT link.artifact_id - FROM job_outputs AS link - JOIN jobs AS job ON job.id = link.job_id - WHERE job.session_id = :session_id - AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) - ) - SELECT hashes.artifact_id, hashes.algorithm, hashes.digest - FROM artifact_hashes AS hashes - JOIN lineage_artifacts ON lineage_artifacts.artifact_id = hashes.artifact_id - ORDER BY hashes.artifact_id, hashes.algorithm, hashes.digest - """ - ), - query_params, - ).mappings() - composite_components = db_ctx.session.execute( - text( - """ - WITH lineage_artifacts AS ( - SELECT link.artifact_id - FROM job_inputs AS link - JOIN jobs AS job ON job.id = link.job_id - WHERE job.session_id = :session_id - AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) - UNION - SELECT link.artifact_id - FROM job_outputs AS link - JOIN jobs AS job ON job.id = link.job_id - WHERE job.session_id = :session_id - AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) - ) - SELECT component.composite_artifact_id, component.ordinal, - component.relative_path, component.leaf_kind, - component.component_algorithm, component.component_digest, - component.component_size, component.component_type - FROM composite_artifact_components AS component - JOIN lineage_artifacts - ON lineage_artifacts.artifact_id = component.composite_artifact_id - ORDER BY component.composite_artifact_id, component.ordinal, - component.relative_path - """ - ), - query_params, - ).mappings() - membership_indexes = db_ctx.session.execute( - text( - """ - WITH lineage_artifacts AS ( - SELECT link.artifact_id - FROM job_inputs AS link - JOIN jobs AS job ON job.id = link.job_id - WHERE job.session_id = :session_id - AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) - UNION - SELECT link.artifact_id - FROM job_outputs AS link - JOIN jobs AS job ON job.id = link.job_id - WHERE job.session_id = :session_id - AND (:exclude_job_uid IS NULL OR job.job_uid IS NULL OR job.job_uid != :exclude_job_uid) - ) - SELECT membership.* - FROM composite_membership_indexes AS membership - JOIN lineage_artifacts - ON lineage_artifacts.artifact_id = membership.composite_artifact_id - ORDER BY membership.composite_artifact_id - """ - ), - query_params, - ).mappings() +def _without_lineage_job(lineage: LineageData, job_uid: str | None) -> LineageData: + if not job_uid: + return lineage + return LineageData( + jobs=[job for job in lineage.jobs if job.get("job_uid") != job_uid], + artifacts=lineage.artifacts, + artifact_hashes=lineage.artifact_hashes, + pipeline=lineage.pipeline, + ) + + +def _collected_lineage_revision(lineage: LineageData) -> str: + """Hash the exact target-rooted lineage snapshot passed to put execution.""" + jobs = sorted( + (_stable_json_value(job) for job in lineage.jobs), + key=lambda item: json.dumps(item, sort_keys=True, separators=(",", ":")), + ) + artifacts = sorted( + (_stable_json_value(artifact) for artifact in lineage.artifacts), + key=lambda item: json.dumps(item, sort_keys=True, separators=(",", ":")), + ) return _fingerprint( { - "jobs": [dict(row) for row in jobs], - "links": [dict(row) for row in links], - "artifacts": [dict(row) for row in artifacts], - "artifact_hashes": [dict(row) for row in artifact_hashes], - "composite_components": [dict(row) for row in composite_components], - "membership_indexes": [dict(row) for row in membership_indexes], + "jobs": jobs, + "artifacts": artifacts, + "artifact_hashes": sorted(lineage.artifact_hashes), } ) +def _stable_json_value(value: Any) -> Any: + if isinstance(value, Mapping): + return {str(key): _stable_json_value(child) for key, child in sorted(value.items())} + if isinstance(value, (list, tuple)): + return [_stable_json_value(child) for child in value] + if isinstance(value, set): + normalized = [_stable_json_value(child) for child in value] + return sorted(normalized, key=lambda child: json.dumps(child, sort_keys=True)) + if isinstance(value, Path): + return str(value) + if isinstance(value, bytes): + return value.hex() + return value + + def _reserve_delegated_put_operation( *, db_ctx: Any, diff --git a/tests/application/publish/test_put_preparation.py b/tests/application/publish/test_put_preparation.py index 08acfe0a..0c1b9543 100644 --- a/tests/application/publish/test_put_preparation.py +++ b/tests/application/publish/test_put_preparation.py @@ -14,6 +14,7 @@ ) from roar.core.interfaces.registration import GitContext from roar.db.context import create_database_context +from roar.db.hashing import hash_files_blake3 from roar.integrations.storage import MemoryBackend @@ -302,19 +303,57 @@ def test_delegated_put_rejects_changed_lineage_while_retry_is_pending( ), ): _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + _record_model_producer(db_ctx, session_id, tmp_path / "model.pt") + db_ctx.commit() + + with pytest.raises(ValueError, match="different delegated put operation"): + _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + + +def test_delegated_put_ignores_unrelated_active_session_jobs_on_retry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _set_delegated_task(monkeypatch) + (tmp_path / "model.pt").write_bytes(b"model") + roar_dir = tmp_path / ".roar" + runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" + git_context = GitContext(repo="repo", branch="main", commit="deadbeef") + + with create_database_context(roar_dir) as db_ctx: + session_id = db_ctx.sessions.get_or_create_active() + db_ctx.commit() + with ( + patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=git_context, + ), + patch( + "roar.application.publish.put_preparation.prepare_publish_session", + return_value=MagicMock( + session_hash="session-hash", + session_url=None, + registration_session_id="registration-session", + registration_session_mode="delegated", + registration_session_status="active", + ), + ), + ): + first = _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) db_ctx.session.execute( text( """ INSERT INTO jobs (timestamp, command, session_id, step_number) - VALUES (1, 'python upstream.py', :session_id, 1) + VALUES (1, 'python unrelated.py', :session_id, 1) """ ), {"session_id": session_id}, ) db_ctx.commit() + retry = _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) - with pytest.raises(ValueError, match="different delegated put operation"): - _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + assert retry.delegated_put_operation == first.delegated_put_operation def test_delegated_put_retry_excludes_its_persisted_sink_and_recovers_closed_receipt( @@ -432,7 +471,7 @@ def test_delegated_put_retry_excludes_its_persisted_sink_and_recovers_closed_rec {"metadata": '{"changed":true}'}, ), ( - "UPDATE job_inputs SET byte_ranges = '[[0,4]]' WHERE job_id = :job_id", + "UPDATE job_outputs SET byte_ranges = '[[0,4]]' WHERE job_id = :job_id", {"job_id": 1}, ), ], @@ -452,11 +491,52 @@ def test_delegated_put_rejects_emitted_lineage_contract_changes( with create_database_context(roar_dir) as db_ctx: session_id = db_ctx.sessions.get_or_create_active() - job_id, _ = db_ctx.jobs.create( - command="python upstream.py", + job_id = _record_model_producer(db_ctx, session_id, tmp_path / "model.pt") + db_ctx.commit() + with ( + patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=git_context, + ), + patch( + "roar.application.publish.put_preparation.prepare_publish_session", + return_value=MagicMock( + session_hash="session-hash", + session_url=None, + registration_session_id="registration-session", + registration_session_mode="delegated", + registration_session_status="active", + ), + ), + ): + _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + bound_params = {**params, "job_id": job_id} + db_ctx.session.execute(text(mutation_sql), bound_params) + db_ctx.commit() + with pytest.raises(ValueError, match="different delegated put operation"): + _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + + +def test_delegated_put_rejects_changed_producer_from_an_earlier_session( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _set_delegated_task(monkeypatch) + model = tmp_path / "model.pt" + model.write_bytes(b"model") + model_digest = hash_files_blake3([model])[str(model)] + roar_dir = tmp_path / ".roar" + runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" + git_context = GitContext(repo="repo", branch="main", commit="deadbeef") + + with create_database_context(roar_dir) as db_ctx: + producer_session_id = db_ctx.sessions.get_or_create_active() + producer_job_id, _ = db_ctx.jobs.create( + command="python produce.py", timestamp=1, - job_uid="upstream", - session_id=session_id, + job_uid="earlier-producer", + session_id=producer_session_id, step_number=1, metadata="{}", ) @@ -464,12 +544,23 @@ def test_delegated_put_rejects_emitted_lineage_contract_changes( text( """ INSERT INTO artifacts (id, size, first_seen_at, first_seen_path) - VALUES ('upstream-artifact', 5, 1, 'input.bin') + VALUES ('model-artifact', 5, 1, 'model.pt') """ ) ) - db_ctx.jobs.add_input(job_id, "upstream-artifact", "input.bin") + db_ctx.session.execute( + text( + """ + INSERT INTO artifact_hashes (artifact_id, algorithm, digest) + VALUES ('model-artifact', 'blake3', :digest) + """ + ), + {"digest": model_digest}, + ) + db_ctx.jobs.add_output(producer_job_id, "model-artifact", "model.pt") + active_session_id = db_ctx.sessions.create(make_active=True) db_ctx.commit() + with ( patch( "roar.application.publish.put_preparation.resolve_roar_git_context", @@ -486,10 +577,14 @@ def test_delegated_put_rejects_emitted_lineage_contract_changes( ), ), ): - _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) - bound_params = {**params, "job_id": job_id} - db_ctx.session.execute(text(mutation_sql), bound_params) + prepared = _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + assert prepared.session_id == active_session_id + db_ctx.session.execute( + text("UPDATE jobs SET metadata = :metadata WHERE id = :job_id"), + {"job_id": producer_job_id, "metadata": '{"changed":true}'}, + ) db_ctx.commit() + with pytest.raises(ValueError, match="different delegated put operation"): _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) @@ -500,6 +595,39 @@ def _set_delegated_task(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("ROAR_DELEGATED_TASK_ID", "task-1") +def _record_model_producer(db_ctx, session_id: int, model: Path) -> int: + digest = hash_files_blake3([model])[str(model)] + artifact_id = f"model-artifact-{session_id}" + job_id, _ = db_ctx.jobs.create( + command="python upstream.py", + timestamp=1, + job_uid="upstream", + session_id=session_id, + step_number=1, + metadata="{}", + ) + db_ctx.session.execute( + text( + """ + INSERT INTO artifacts (id, size, first_seen_at, first_seen_path) + VALUES (:artifact_id, 5, 1, 'model.pt') + """ + ), + {"artifact_id": artifact_id}, + ) + db_ctx.session.execute( + text( + """ + INSERT INTO artifact_hashes (artifact_id, algorithm, digest) + VALUES (:artifact_id, 'blake3', :digest) + """ + ), + {"artifact_id": artifact_id, "digest": digest}, + ) + db_ctx.jobs.add_output(job_id, artifact_id, "model.pt") + return job_id + + def _prepare_model_put(db_ctx, runtime, roar_dir: Path, repo_root: Path) -> PreparedPutExecution: return prepare_put_execution( db_ctx=db_ctx, diff --git a/tests/unit/put/test_put_service.py b/tests/unit/put/test_put_service.py index ba87df71..48970807 100644 --- a/tests/unit/put/test_put_service.py +++ b/tests/unit/put/test_put_service.py @@ -86,6 +86,7 @@ def _prepared_put( registration_session_id: str | None = None, registration_session_status: str | None = None, delegated_put_operation: DelegatedPutOperation | None = None, + lineage: LineageData | None = None, ) -> PreparedPutExecution: resolved: list[ResolvedSource] = [] for source in sources: @@ -126,10 +127,40 @@ def _prepared_put( registration_session_id=registration_session_id, registration_session_status=registration_session_status, delegated_put_operation=delegated_put_operation, + lineage=lineage, ) class TestPutService: + def test_put_prepared_uses_the_reserved_lineage_snapshot(self, tmp_path: Path) -> None: + model_file = tmp_path / "model.pt" + model_file.write_bytes(b"model data") + lineage = LineageData( + jobs=[{"job_uid": "reserved-upstream", "command": "python upstream.py"}], + artifacts=[], + artifact_hashes=set(), + ) + collector = MagicMock() + coordinator = _create_mock_coordinator() + service = PutService( + db_context=_create_mock_db(), + backend=MemoryBackend(bucket="test-bucket", prefix="models"), + destination="memory://test-bucket/models", + repo_root=tmp_path, + lineage_collector=collector, + registration_coordinator=coordinator, + ) + + result = service.put_prepared( + prepared=_prepared_put(tmp_path, sources=[model_file], lineage=lineage), + sources=[str(model_file)], + message="publish reserved snapshot", + ) + + assert result.success is True + collector.collect.assert_not_called() + assert coordinator.register_lineage.call_args.kwargs["jobs"] == lineage.jobs + def test_closed_registration_session_retry_does_not_upload_again(self, tmp_path: Path) -> None: model_file = tmp_path / "model.pt" model_file.write_bytes(b"model data") From 2519fdaaf1d9098452069e2a9f2c0efb8f58b794 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Fri, 7 Aug 2026 22:11:42 +0000 Subject: [PATCH 30/52] tracker: record packages from dist-packages, not just site-packages (P0-18) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_used_packages's file pass matched only "site-packages", so a package installed under dist-packages (Debian/Ubuntu system Python — the cert AMIs) was dropped from the freeze entirely: the workload imports it, but the file pass skips it and the aliased-only name pass (P0-13 fix) doesn't rescue a normally-loaded import. Row 007 recorded only brotli+zstandard (venv/site-packages, pulled during the download) while huggingface-hub/tqdm/... (system/dist-packages) vanished — a thin freeze that passes every gate and rebuilds only by luck via transitive resolution. Recognize "dist-packages/" alongside "site-packages/" (via _site_packages_top, now used by the file pass too). This is the mirror of P0-13: #264's broad name pass masked this, #267's aliased-only narrowing re-exposed it. Does NOT reintroduce P0-13: the file pass records only genuinely-LOADED modules (sys.modules), so a probed import that FAILED (e.g. an unsatisfiable sagemaker on the AMI) is never in modules_files and stays out. Tests cover both: a dist-packages import is recorded; a failed probe (name-only, not loaded) is not. Note: safe iff the P0-13 probe imports FAIL (don't load). If a probe imports SUCCESSFULLY on the AMI it would be recorded again — pending MMA confirmation of sagemaker's load state on the row-008 capture. Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/execution/runtime/inject/tracker.py | 44 +++++++++---------- .../runtime/test_used_packages_by_name.py | 28 ++++++++++++ 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/roar/execution/runtime/inject/tracker.py b/roar/execution/runtime/inject/tracker.py index dc5c66fb..3e9dc38b 100644 --- a/roar/execution/runtime/inject/tracker.py +++ b/roar/execution/runtime/inject/tracker.py @@ -90,17 +90,27 @@ def _dist_is_in_repo(dist_name: str, repo_root: str) -> bool: return False +# Package install roots. "dist-packages" (Debian/Ubuntu system Python, e.g. the +# cert AMIs) must be recognized alongside "site-packages" — otherwise packages +# installed there are dropped from the freeze entirely (P0-18): the workload +# imports them, but the file pass ignored them and the aliased-only name pass +# (P0-13 fix) doesn't rescue a normally-loaded import. +_PACKAGE_ROOT_MARKERS = ("site-packages/", "dist-packages/") + + def _site_packages_top(fpath: str) -> str | None: - """The top-level package dir for a file under site-packages, else None.""" - idx = fpath.find("site-packages/") - if idx < 0: - return None - top = fpath[idx + len("site-packages/") :].split("/")[0] - if top.endswith(".py"): - top = top[:-3] - if top.startswith("_") or top.endswith((".dist-info", ".egg-info", ".so")): - return None - return top + """The top-level package dir for a file under a package install root, else None.""" + for marker in _PACKAGE_ROOT_MARKERS: + idx = fpath.find(marker) + if idx < 0: + continue + top = fpath[idx + len(marker) :].split("/")[0] + if top.endswith(".py"): + top = top[:-3] + if top.startswith("_") or top.endswith((".dist-info", ".egg-info", ".so")): + return None + return top + return None def get_used_packages( @@ -123,18 +133,8 @@ def get_used_packages( try: for fpath in modules_files: - if "site-packages" not in fpath: - continue - idx = fpath.find("site-packages/") - if idx < 0: - continue - after_sp = fpath[idx + len("site-packages/") :] - top_dir = after_sp.split("/")[0] - if top_dir.endswith(".py"): - top_dir = top_dir[:-3] - if top_dir.endswith(".dist-info") or top_dir.endswith(".egg-info"): - continue - if top_dir.startswith("_") or top_dir.endswith(".so"): + top_dir = _site_packages_top(fpath) + if top_dir is None: continue if top_dir == "roar": # roar records itself otherwise. roar-cli is installed separately diff --git a/tests/execution/runtime/test_used_packages_by_name.py b/tests/execution/runtime/test_used_packages_by_name.py index e7889dc4..990c0d21 100644 --- a/tests/execution/runtime/test_used_packages_by_name.py +++ b/tests/execution/runtime/test_used_packages_by_name.py @@ -178,3 +178,31 @@ def test_real_aliased_dep_outside_repo_is_still_pinned(tmp_path, monkeypatch): workload_root=str(repo), ) assert used.get("wandb") == "0.16.0" + + +def test_dist_packages_import_is_recorded(monkeypatch): + """P0-18: a package loaded from dist-packages (system Python, e.g. the cert + AMI) must reach the freeze — the file pass previously only matched + site-packages, so it was dropped entirely.""" + monkeypatch.setattr( + ilm, "packages_distributions", lambda: {"huggingface_hub": ["huggingface-hub"]} + ) + used = get_used_packages( + modules_files=["/usr/lib/python3/dist-packages/huggingface_hub/__init__.py"], + installed_packages={"huggingface-hub": "1.27.0"}, + ) + assert used.get("huggingface-hub") == "1.27.0" + + +def test_failed_probe_import_is_not_recorded_even_with_dist_packages(monkeypatch): + """P0-13 must stay fixed: a probed import that FAILED (never loaded, so not in + modules_files/loaded_files — only its name is in imported_modules) is not + recorded, even now that dist-packages is recognized.""" + monkeypatch.setattr(ilm, "packages_distributions", lambda: {"sagemaker": ["sagemaker-core"]}) + used = get_used_packages( + modules_files=[], # import sagemaker failed -> not loaded + installed_packages={"sagemaker-core": "2.9.0"}, + imported_modules=["sagemaker"], # name recorded before the failed import + loaded_files={}, # not in sys.modules + ) + assert "sagemaker-core" not in used From 03a60435ad21212369fd40fbff7991bc80645856 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Fri, 7 Aug 2026 22:02:08 +0000 Subject: [PATCH 31/52] security: redact HF tokens by default (P0-16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HF_TOKEN was not in roar's default env-var redaction list, and the campaign runs --lineage public with `secrets: - HF_TOKEN` declared by design — so a live token could publish verbatim in job metadata. (MMA swept 10 DAGs: 0 leaked, by luck.) - Add HF_TOKEN + HUGGING_FACE_HUB_TOKEN to the built-in env-var name defaults (schema, raw, access) AND the `roar init` template — config resolution replaces the names list, so the template must carry them too or a fresh init would drop them. - Broaden the always-on HF value pattern in filters.omit.BUILTIN_PATTERNS from exact `hf_[a-zA-Z0-9]{34}` to `hf_[A-Za-z0-9]{20,}` so length variants are still caught. BUILTIN_PATTERNS apply unconditionally and can't be disabled by config, which is why the value regex lives there rather than as a config default. Tests: HF_TOKEN/HUGGING_FACE_HUB_TOKEN are in the default env-var names; the value pattern catches a 30-char token (the old exact-34 regex missed it). Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/cli/commands/init.py | 3 +++ roar/filters/omit.py | 4 ++-- roar/integrations/config/access.py | 2 ++ roar/integrations/config/raw.py | 2 ++ roar/integrations/config/schema.py | 4 ++++ tests/unit/test_register_secrets.py | 23 +++++++++++++++++++++++ 6 files changed, 36 insertions(+), 2 deletions(-) diff --git a/roar/cli/commands/init.py b/roar/cli/commands/init.py index e0645a66..9e45ce1d 100644 --- a/roar/cli/commands/init.py +++ b/roar/cli/commands/init.py @@ -81,7 +81,10 @@ "GITHUB_TOKEN", "DATABASE_URL", "AWS_SECRET_ACCESS_KEY", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", ] +# (HF token *values* are also caught unconditionally by roar's built-in patterns.) [registration.omit.allowlist] # Regex patterns that should NOT be redacted (reduce false positives) diff --git a/roar/filters/omit.py b/roar/filters/omit.py index 7c3b02ce..ebd85356 100644 --- a/roar/filters/omit.py +++ b/roar/filters/omit.py @@ -77,10 +77,10 @@ def was_modified(self) -> bool: re.compile(r"(sk-ant-[a-zA-Z0-9\-]+)"), "[ANTHROPIC_KEY_REDACTED]", ), - # HuggingFace token + # HuggingFace token — {20,} (not exact-34) to catch token-length variants (P0-16) ( "huggingface_token", - re.compile(r"(hf_[a-zA-Z0-9]{34})"), + re.compile(r"(hf_[A-Za-z0-9]{20,})"), "[HF_TOKEN_REDACTED]", ), # GitLab personal/project/CI access tokens diff --git a/roar/integrations/config/access.py b/roar/integrations/config/access.py index db67cb5f..ee1475a0 100644 --- a/roar/integrations/config/access.py +++ b/roar/integrations/config/access.py @@ -131,6 +131,8 @@ "GITHUB_TOKEN", "DATABASE_URL", "AWS_SECRET_ACCESS_KEY", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", ], "description": "Env var names whose values should be redacted (comma-separated)", }, diff --git a/roar/integrations/config/raw.py b/roar/integrations/config/raw.py index f4c85d48..ab3a968b 100644 --- a/roar/integrations/config/raw.py +++ b/roar/integrations/config/raw.py @@ -28,6 +28,8 @@ "GITHUB_TOKEN", "DATABASE_URL", "AWS_SECRET_ACCESS_KEY", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", ] }, "patterns": [], diff --git a/roar/integrations/config/schema.py b/roar/integrations/config/schema.py index ab29c617..f853b93f 100644 --- a/roar/integrations/config/schema.py +++ b/roar/integrations/config/schema.py @@ -109,6 +109,8 @@ class EnvVarsConfig(ConfigBaseModel): "GITHUB_TOKEN", "DATABASE_URL", "AWS_SECRET_ACCESS_KEY", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", ] ) @@ -133,6 +135,8 @@ class OmitConfig(ConfigBaseModel): enabled: bool = True secrets: SecretsConfig = Field(default_factory=SecretsConfig) env_vars: EnvVarsConfig = Field(default_factory=EnvVarsConfig) + # Value regexes (incl. the HF token) live in filters.omit.BUILTIN_PATTERNS, + # which is applied unconditionally and can't be disabled by config. patterns: list[CustomPattern] = Field(default_factory=list) allowlist: AllowlistConfig = Field(default_factory=AllowlistConfig) diff --git a/tests/unit/test_register_secrets.py b/tests/unit/test_register_secrets.py index 68b510dc..49c56449 100644 --- a/tests/unit/test_register_secrets.py +++ b/tests/unit/test_register_secrets.py @@ -142,3 +142,26 @@ def test_filter_git_context_secrets_without_filter_returns_context_unchanged() - assert filtered is context assert detections == [] + + +def test_hf_token_env_vars_are_redacted_by_default() -> None: + """P0-16: HF_TOKEN / HUGGING_FACE_HUB_TOKEN must be in the built-in env-var + redaction defaults so a live token doesn't reach a published DAG.""" + from roar.filters.omit import OmitFilter + from roar.integrations.config.raw import _DEFAULT_REGISTRATION_OMIT + + names = OmitFilter(_DEFAULT_REGISTRATION_OMIT).env_var_names + assert "HF_TOKEN" in names + assert "HUGGING_FACE_HUB_TOKEN" in names + + +def test_hf_token_value_regex_catches_length_variants() -> None: + """P0-16: the always-on HF value pattern is hf_[A-Za-z0-9]{20,}, not exact-34, + so a token that isn't 34 chars is still caught (defense-in-depth).""" + from roar.filters.omit import OmitFilter + + f = OmitFilter({}) # BUILTIN_PATTERNS apply regardless of config + ids_30 = {m.pattern_id for m in f.detect_secrets("token=hf_" + "A" * 30)} + ids_34 = {m.pattern_id for m in f.detect_secrets("token=hf_" + "A" * 34)} + assert "huggingface_token" in ids_30 # 30 chars: the old {34} regex MISSED this + assert "huggingface_token" in ids_34 From e2ee09f7860286880bfee35f51df4cfe38cdacb2 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Fri, 7 Aug 2026 21:41:55 +0000 Subject: [PATCH 32/52] fix: cold-host wandb-trackio stub crash (P0-15) + huggingface-hub runtime dep (P0-19) P0-15: --wandb-to-trackio's no-op stub (used on any credential-free host, i.e. every cold reproduce host) was built with types.ModuleType, leaving __spec__ = None. importlib.util.find_spec RAISES on that, and accelerate's is_wandb_available() calls it at `import accelerate` -> ValueError: wandb.__spec__ is None. So any row importing accelerate/transformers/diffusers under --wandb-to-trackio crashed at import on a cold host (never on capture). Give the stub a real ModuleSpec. P0-19: `roar put hf://` imports huggingface_hub, but it was declared only under the `dev` extra, so `uv tool install ` (the mandated install) omitted it and put failed at publish time -- after every traced step had run (~6.5 GPU-min wasted on row 007). Move huggingface-hub to runtime dependencies. Tests: no-op stub carries a spec so find_spec doesn't raise (P0-15); huggingface-hub is in roar-cli's runtime requires, not behind an extra (P0-19). Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 2 +- roar/integrations/wandb_trackio.py | 7 +++++++ tests/integrations/test_wandb_trackio.py | 13 +++++++++++++ tests/unit/test_runtime_dependencies.py | 19 +++++++++++++++++++ 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_runtime_dependencies.py diff --git a/pyproject.toml b/pyproject.toml index 83f69f95..87ac9221 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ dependencies = [ "pydantic-settings>=2.0.0", "textual>=0.80", "tomli>=2.0.0; python_version < '3.11'", + "huggingface_hub>=0.20.0", # `roar put hf://` is a shipped path, not dev-only (P0-19) ] [project.urls] @@ -68,7 +69,6 @@ dev = [ "mypy>=1.13.0", "boto3>=1.28.0", "google-cloud-storage>=2.10.0", - "huggingface_hub>=0.20.0", ] [project.scripts] diff --git a/roar/integrations/wandb_trackio.py b/roar/integrations/wandb_trackio.py index c432a8dc..7688af18 100644 --- a/roar/integrations/wandb_trackio.py +++ b/roar/integrations/wandb_trackio.py @@ -174,9 +174,16 @@ def _f(*a, **k): def _install_noop_wandb() -> None: """Alias ``wandb`` to a silent no-op module so an unmodified repo runs untracked.""" + import importlib.machinery import types mod = types.ModuleType("wandb") + # types.ModuleType leaves __spec__ = None, and importlib.util.find_spec RAISES + # ("wandb.__spec__ is None") rather than returning None on that. accelerate's + # is_wandb_available() calls find_spec at `import accelerate`, so a credential- + # free host (i.e. every cold reproduce host) crashes on import. Give the stub a + # real spec. P0-15. + mod.__spec__ = importlib.machinery.ModuleSpec("wandb", loader=None) def _noop(*a, **k): return None diff --git a/tests/integrations/test_wandb_trackio.py b/tests/integrations/test_wandb_trackio.py index 7d132391..5e74d111 100644 --- a/tests/integrations/test_wandb_trackio.py +++ b/tests/integrations/test_wandb_trackio.py @@ -84,3 +84,16 @@ def test_does_not_clobber_existing_wandb(): sys.modules["wandb"] = sentinel wandb_trackio.install(environ={"ROAR_WANDB_TO_TRACKIO": "off"}) assert sys.modules["wandb"] is sentinel + + +def test_noop_wandb_has_a_spec_so_find_spec_does_not_raise(): + """P0-15: a __spec__=None module makes importlib.util.find_spec RAISE, which + crashes `import accelerate` (is_wandb_available) on any credential-free host. + The no-op stub must carry a real spec.""" + import importlib.util + + wandb_trackio.install(environ={"ROAR_WANDB_TO_TRACKIO": "off"}) + stub = sys.modules["wandb"] + assert stub.__spec__ is not None + # This raised ValueError("wandb.__spec__ is None") before the fix. + assert importlib.util.find_spec("wandb") is stub.__spec__ diff --git a/tests/unit/test_runtime_dependencies.py b/tests/unit/test_runtime_dependencies.py new file mode 100644 index 00000000..e29149ec --- /dev/null +++ b/tests/unit/test_runtime_dependencies.py @@ -0,0 +1,19 @@ +"""Guard roar's declared runtime dependencies for shipped code paths.""" + +from __future__ import annotations + +from importlib import metadata + + +def test_huggingface_hub_is_a_runtime_dependency(): + """P0-19: `roar put hf://` imports huggingface_hub, so it must be a RUNTIME + dependency, not a dev-only extra. When it lived under the `dev` extra, + `uv tool install ` (the mandated install path) omitted it and + `roar put hf://` died with ModuleNotFoundError *after* every step succeeded. + """ + reqs = metadata.requires("roar-cli") or [] + hf = [r for r in reqs if r.lower().replace("_", "-").startswith("huggingface-hub")] + assert hf, f"huggingface_hub missing from roar-cli requirements: {reqs}" + # It must not be gated behind an extra (e.g. `; extra == "dev"`). + behind_extra = [r for r in hf if "extra ==" in r or "extra==" in r] + assert not behind_extra, f"huggingface_hub is still behind an extra: {behind_extra}" From 3f6e4dc7eb5c570597d7c516c06a330e6828a727 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Mon, 10 Aug 2026 14:49:19 +0000 Subject: [PATCH 33/52] P0-18/P0-11: attribute the freeze to the workload, not roar's footprint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When roar shares the workload's venv, roar's own dependencies (pydantic, click, blake3, cryptography, the pydantic tree, ...) load from the same site-packages as the workload's and the file pass pinned them into the freeze as if the job needed them. That is the "roar footprint" that lets a thin freeze rebuild by luck (P0-11 broad / P0-18): roar's over-broad pins fill gaps a truly-closed workload freeze would expose. Subtract roar's footprint from the freeze WITHOUT dropping any package the workload actually uses, using three signals combined: - ORIGIN attribution (tracking_import): a top-level name is workload-owned when it is imported from the workload's own loose code (entry script / local modules / editable repo), and roar-owned when imported from a roar.* module. A cache hit still calls __import__ with the requesting module's globals, so `import yaml` from the workload is observed even when roar loaded yaml first at injection — which timing/presence on sys.modules cannot see. An installed package importing itself (pydantic -> pydantic) lives under a package root, so it can't masquerade as a workload need. - INSTALL BASELINE: sys.modules keys present when install() runs are roar's bootstrap footprint (loaded before __import__ is patched). - DEPENDENCY CLOSURE: roar-cli's transitive runtime deps from installed metadata, catching deps roar pulls in via importlib (the ABI gate's pydantic probe) that neither hook nor baseline observe. roar_exclusive = (origin-roar | install-baseline | roar-closure) MINUS anything the workload imported. The workload subtraction is what keeps a shared dependency — e.g. pyyaml on a timm row — in the freeze even though roar depends on and loaded it too. Exercised end-to-end through the real sitecustomize across install shapes: shared (freeze 9->1 pkgs, footprint gone), shared-with-collision (workload imports pydantic -> pydantic survives), isolated (already clean, no regression), version mismatch (records the workload's pyyaml 6.0.1, not roar's 6.0.3; no sys.modules shadow), and editable (P0-12 self-pkg still unpinned). Existing unit tests unchanged. pkg_dist_map is built once and reused (atexit overhead), and the install-time snapshot is just a key-set copy off the hot start path. Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/execution/runtime/inject/tracker.py | 173 +++++++++++++++++++++-- 1 file changed, 165 insertions(+), 8 deletions(-) diff --git a/roar/execution/runtime/inject/tracker.py b/roar/execution/runtime/inject/tracker.py index 3e9dc38b..cbeaf6f0 100644 --- a/roar/execution/runtime/inject/tracker.py +++ b/roar/execution/runtime/inject/tracker.py @@ -98,6 +98,75 @@ def _dist_is_in_repo(dist_name: str, repo_root: str) -> bool: _PACKAGE_ROOT_MARKERS = ("site-packages/", "dist-packages/") +# Standard-library / builtin top-level names never belong in a package freeze, +# and must not be attributed as roar-only (which would let a stdlib name roar +# imported shadow a same-named workload package). Bootstrap-safe: available since +# the interpreter starts. +_STDLIB = frozenset(getattr(sys, "stdlib_module_names", frozenset())) + + +def _normalize_dist(name: str) -> str: + return name.lower().replace("_", "-") + + +def _req_dist_name(requirement: str) -> str: + """The distribution name at the head of a Requires-Dist string, e.g. + ``pydantic>=2.0.0`` -> ``pydantic``; ``click (>=8.1)`` -> ``click``.""" + head = requirement.strip() + for sep in (";", " ", "[", "(", "<", ">", "=", "!", "~"): + idx = head.find(sep) + if idx >= 0: + head = head[:idx] + return head.strip() + + +def roar_dependency_closure(root: str = "roar-cli") -> set[str]: + """Transitive runtime-dependency distribution names of ``root`` (normalized), + read from installed metadata. Extras-gated (dev/test) deps are skipped so the + freeze isn't scrubbed of a package that only *shares a name* with a dev extra. + Best-effort: returns an empty set on any failure, so exclusion degrades to the + frame/baseline signals rather than dropping anything unexpectedly.""" + try: + from importlib import metadata as importlib_metadata + except Exception: + return set() + seen: set[str] = set() + stack = [root] + while stack: + name = _normalize_dist(stack.pop()) + if name in seen: + continue + seen.add(name) + try: + requirements = importlib_metadata.requires(name) or [] + except Exception: + continue + for requirement in requirements: + if "extra ==" in requirement or "extra==" in requirement: + continue + dep = _req_dist_name(requirement) + if dep: + stack.append(dep) + seen.discard(_normalize_dist(root)) + return seen + + +def _installed_top_names(modules: Any) -> set[str]: + """Top-level import names of the given installed (site-/dist-packages) modules. + ``__file__`` is used raw — ``_site_packages_top`` keys on a path substring, so + no abspath is needed (that call is per-module and this runs on the hot start + path). Used to resolve roar's bootstrap footprint from the install snapshot.""" + names: set[str] = set() + for module in modules: + fpath = getattr(module, "__file__", None) + if not fpath: + continue + top = _site_packages_top(fpath) + if top and top != "roar": + names.add(top) + return names + + def _site_packages_top(fpath: str) -> str | None: """The top-level package dir for a file under a package install root, else None.""" for marker in _PACKAGE_ROOT_MARKERS: @@ -119,17 +188,29 @@ def get_used_packages( imported_modules: Sequence[str] = (), workload_root: str | None = None, loaded_files: Mapping[str, str] | None = None, + roar_exclusive_names: Sequence[str] = (), + pkg_dist_map: Mapping[str, list[str]] | None = None, ) -> dict[str, str | None]: used: dict[str, str | None] = {} repo_root = os.path.abspath(workload_root) if workload_root else None loaded = loaded_files or {} + # Top-level import names that ONLY roar's own machinery imported (never the + # workload). When roar shares the workload's venv, roar's deps (pydantic, + # click, blake3, ...) load from the same site-packages as the workload's and + # the file pass would otherwise pin them as if the job needed them — P0-11 / + # P0-18's "roar footprint" that makes a thin freeze rebuild by luck. A name is + # only here when NO non-roar module imported it (see tracking_import), so a + # package the workload genuinely uses — even one roar also imports, e.g. + # pyyaml on a timm row — is never dropped. + roar_exclusive = set(roar_exclusive_names) + + if pkg_dist_map is None: + try: + from importlib import metadata as importlib_metadata - try: - from importlib import metadata as importlib_metadata - - pkg_dist_map = importlib_metadata.packages_distributions() - except Exception: - pkg_dist_map = {} + pkg_dist_map = importlib_metadata.packages_distributions() + except Exception: + pkg_dist_map = {} try: for fpath in modules_files: @@ -142,6 +223,9 @@ def get_used_packages( # harmless noise on a PyPI release, but fatal on an unpublished # build (roar-cli==X.Y.dev0 can't resolve). P0-11. continue + if top_dir in roar_exclusive: + # A dependency only roar's own machinery imported (P0-11 broad). + continue pkg_names = pkg_dist_map.get(top_dir, []) for pkg_name in pkg_names: @@ -173,7 +257,7 @@ def get_used_packages( try: for name in imported_modules: top = name.split(".")[0] - if not top or top.startswith("_") or top == "roar": + if not top or top.startswith("_") or top == "roar" or top in roar_exclusive: continue loaded_file = loaded.get(top) if not loaded_file: @@ -195,7 +279,13 @@ def get_used_packages( return used -_MERGE_LIST_FIELDS = ("opened_files", "imported_modules", "modules_files", "shared_libs") +_MERGE_LIST_FIELDS = ( + "opened_files", + "imported_modules", + "modules_files", + "shared_libs", + "roar_exclusive", +) _MERGE_DICT_FIELDS = ("used_packages", "installed_packages", "env_reads") @@ -295,6 +385,19 @@ def __init__( self.opened_files: set[str] = set() self.imported_modules: set[str] = set() self.env_reads: dict[str, str] = {} + # Import attribution by ORIGIN (the importing module's __name__): a + # top-level name goes to workload_import_names if ANY non-roar module + # imported it, and to roar_import_names when a roar.* module did. Their + # difference (roar-only) is subtracted from the freeze so roar's own + # dependency footprint doesn't masquerade as the workload's. P0-11/P0-18. + self.workload_import_names: set[str] = set() + self.roar_import_names: set[str] = set() + # sys.modules keys present when install() runs — roar's own bootstrap + # footprint (roar + the deps it imports to build the tracker, before + # __import__ is patched, so tracking_import can't attribute them). Keys are + # snapshotted cheaply at install; their top-level names are resolved at + # write_log (off the hot start path). + self._install_baseline_keys: set[str] = set() if not hasattr(self._environ, _ORIGINAL_ENVIRON_GET_ATTR): with contextlib.suppress(Exception): @@ -304,6 +407,10 @@ def __init__( def install(self) -> None: """Patch builtins and environ access for activity capture.""" + # Everything loaded right now is roar's bootstrap footprint: the workload + # hasn't run yet. Snapshot the keys cheaply (a set copy) before patching + # __import__ so it covers deps roar imported to build the tracker itself. + self._install_baseline_keys = set(sys.modules) builtins.open = self.tracking_open builtins.__import__ = self.tracking_import setattr(self._environ, _ENVIRON_GET_METHOD_NAME, self.patched_environ_get) @@ -320,6 +427,25 @@ def tracking_open(self, *args, **kwargs): def tracking_import(self, name, globals=None, locals=None, fromlist=(), level=0): self.imported_modules.add(name) + # Attribute this import by its ORIGIN. A cache hit still calls __import__ + # with the requesting module's globals, so `import yaml` issued from the + # workload is observed here even when roar loaded yaml first at injection — + # which is exactly what a presence/timing check on sys.modules cannot see. + if level == 0 and name: + top = name.split(".")[0] + if top and not top.startswith("_") and top not in _STDLIB: + origin = (globals or {}).get("__name__") or "" + if origin == "roar" or origin.startswith("roar."): + self.roar_import_names.add(top) + else: + origin_file = (globals or {}).get("__file__") + if origin_file and _site_packages_top(origin_file) is None: + # Only the workload's OWN loose code — the entry script, + # local modules, an editable repo — protects a name from + # roar-dependency exclusion. An installed package's internal + # import (e.g. pydantic importing itself) lives under a + # package root, so it can't masquerade as a workload need. + self.workload_import_names.add(top) module = self._real_import(name, globals, locals, fromlist, level) if self._environ.get("ROAR_WRAP") != "1": @@ -359,12 +485,42 @@ def write_log(self) -> None: for name, module in sys.modules.items() if getattr(module, "__file__", None) } + # roar's declared dependency closure, mapped from distribution names to the + # import names the file pass keys on. Catches deps roar pulls in via + # importlib (e.g. the ABI gate importing pydantic) that neither the origin + # hook nor the install baseline can observe. + closure_dists = roar_dependency_closure() + try: + from importlib import metadata as importlib_metadata + + pkg_dist_map = importlib_metadata.packages_distributions() + except Exception: + pkg_dist_map = {} + closure_imports = { + imp + for imp, dists in pkg_dist_map.items() + if any(_normalize_dist(d) in closure_dists for d in dists) + } # pkg_dist_map is reused by get_used_packages below (built once). + # roar-only footprint = what roar imported (post-install, by origin), what + # was already loaded at install time (bootstrap), and roar's declared + # dependency closure — MINUS anything the workload itself imported. That + # last subtraction is what keeps a shared dependency (e.g. pyyaml on a timm + # row) in the freeze even though roar depends on and loaded it too. + # P0-11/P0-18. + install_baseline = _installed_top_names( + sys.modules[k] for k in self._install_baseline_keys if k in sys.modules + ) + roar_exclusive = ( + self.roar_import_names | install_baseline | closure_imports + ) - self.workload_import_names used_packages = get_used_packages( modules_files, installed_packages, sorted(self.imported_modules), workload_root=os.getcwd(), loaded_files=loaded_files, + roar_exclusive_names=sorted(roar_exclusive), + pkg_dist_map=pkg_dist_map, ) data = { "opened_files": sorted(self.opened_files), @@ -372,6 +528,7 @@ def write_log(self) -> None: "env_reads": dict(sorted(self.env_reads.items())), "modules_files": modules_files, "roar_inject_dir": self._inject_dir, + "roar_exclusive": sorted(roar_exclusive), "shared_libs": get_loaded_shared_libs(self._real_open), "sys_prefix": sys.prefix, "sys_base_prefix": sys.base_prefix, From 71133868742b648d778d0eb11e25887c645cf09f Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Mon, 10 Aug 2026 20:34:40 +0000 Subject: [PATCH 34/52] Revert "Merge pull request #275 from treqs/p0-18/frame-attribution" This reverts commit 710545ce186ff526ef8b9998e74df4ba5529dc26, reversing changes made to 8ff26042f1fc52aafeafaf7b4386e609d2088e8a. --- roar/execution/runtime/inject/tracker.py | 173 ++--------------------- 1 file changed, 8 insertions(+), 165 deletions(-) diff --git a/roar/execution/runtime/inject/tracker.py b/roar/execution/runtime/inject/tracker.py index cbeaf6f0..3e9dc38b 100644 --- a/roar/execution/runtime/inject/tracker.py +++ b/roar/execution/runtime/inject/tracker.py @@ -98,75 +98,6 @@ def _dist_is_in_repo(dist_name: str, repo_root: str) -> bool: _PACKAGE_ROOT_MARKERS = ("site-packages/", "dist-packages/") -# Standard-library / builtin top-level names never belong in a package freeze, -# and must not be attributed as roar-only (which would let a stdlib name roar -# imported shadow a same-named workload package). Bootstrap-safe: available since -# the interpreter starts. -_STDLIB = frozenset(getattr(sys, "stdlib_module_names", frozenset())) - - -def _normalize_dist(name: str) -> str: - return name.lower().replace("_", "-") - - -def _req_dist_name(requirement: str) -> str: - """The distribution name at the head of a Requires-Dist string, e.g. - ``pydantic>=2.0.0`` -> ``pydantic``; ``click (>=8.1)`` -> ``click``.""" - head = requirement.strip() - for sep in (";", " ", "[", "(", "<", ">", "=", "!", "~"): - idx = head.find(sep) - if idx >= 0: - head = head[:idx] - return head.strip() - - -def roar_dependency_closure(root: str = "roar-cli") -> set[str]: - """Transitive runtime-dependency distribution names of ``root`` (normalized), - read from installed metadata. Extras-gated (dev/test) deps are skipped so the - freeze isn't scrubbed of a package that only *shares a name* with a dev extra. - Best-effort: returns an empty set on any failure, so exclusion degrades to the - frame/baseline signals rather than dropping anything unexpectedly.""" - try: - from importlib import metadata as importlib_metadata - except Exception: - return set() - seen: set[str] = set() - stack = [root] - while stack: - name = _normalize_dist(stack.pop()) - if name in seen: - continue - seen.add(name) - try: - requirements = importlib_metadata.requires(name) or [] - except Exception: - continue - for requirement in requirements: - if "extra ==" in requirement or "extra==" in requirement: - continue - dep = _req_dist_name(requirement) - if dep: - stack.append(dep) - seen.discard(_normalize_dist(root)) - return seen - - -def _installed_top_names(modules: Any) -> set[str]: - """Top-level import names of the given installed (site-/dist-packages) modules. - ``__file__`` is used raw — ``_site_packages_top`` keys on a path substring, so - no abspath is needed (that call is per-module and this runs on the hot start - path). Used to resolve roar's bootstrap footprint from the install snapshot.""" - names: set[str] = set() - for module in modules: - fpath = getattr(module, "__file__", None) - if not fpath: - continue - top = _site_packages_top(fpath) - if top and top != "roar": - names.add(top) - return names - - def _site_packages_top(fpath: str) -> str | None: """The top-level package dir for a file under a package install root, else None.""" for marker in _PACKAGE_ROOT_MARKERS: @@ -188,29 +119,17 @@ def get_used_packages( imported_modules: Sequence[str] = (), workload_root: str | None = None, loaded_files: Mapping[str, str] | None = None, - roar_exclusive_names: Sequence[str] = (), - pkg_dist_map: Mapping[str, list[str]] | None = None, ) -> dict[str, str | None]: used: dict[str, str | None] = {} repo_root = os.path.abspath(workload_root) if workload_root else None loaded = loaded_files or {} - # Top-level import names that ONLY roar's own machinery imported (never the - # workload). When roar shares the workload's venv, roar's deps (pydantic, - # click, blake3, ...) load from the same site-packages as the workload's and - # the file pass would otherwise pin them as if the job needed them — P0-11 / - # P0-18's "roar footprint" that makes a thin freeze rebuild by luck. A name is - # only here when NO non-roar module imported it (see tracking_import), so a - # package the workload genuinely uses — even one roar also imports, e.g. - # pyyaml on a timm row — is never dropped. - roar_exclusive = set(roar_exclusive_names) - - if pkg_dist_map is None: - try: - from importlib import metadata as importlib_metadata - pkg_dist_map = importlib_metadata.packages_distributions() - except Exception: - pkg_dist_map = {} + try: + from importlib import metadata as importlib_metadata + + pkg_dist_map = importlib_metadata.packages_distributions() + except Exception: + pkg_dist_map = {} try: for fpath in modules_files: @@ -223,9 +142,6 @@ def get_used_packages( # harmless noise on a PyPI release, but fatal on an unpublished # build (roar-cli==X.Y.dev0 can't resolve). P0-11. continue - if top_dir in roar_exclusive: - # A dependency only roar's own machinery imported (P0-11 broad). - continue pkg_names = pkg_dist_map.get(top_dir, []) for pkg_name in pkg_names: @@ -257,7 +173,7 @@ def get_used_packages( try: for name in imported_modules: top = name.split(".")[0] - if not top or top.startswith("_") or top == "roar" or top in roar_exclusive: + if not top or top.startswith("_") or top == "roar": continue loaded_file = loaded.get(top) if not loaded_file: @@ -279,13 +195,7 @@ def get_used_packages( return used -_MERGE_LIST_FIELDS = ( - "opened_files", - "imported_modules", - "modules_files", - "shared_libs", - "roar_exclusive", -) +_MERGE_LIST_FIELDS = ("opened_files", "imported_modules", "modules_files", "shared_libs") _MERGE_DICT_FIELDS = ("used_packages", "installed_packages", "env_reads") @@ -385,19 +295,6 @@ def __init__( self.opened_files: set[str] = set() self.imported_modules: set[str] = set() self.env_reads: dict[str, str] = {} - # Import attribution by ORIGIN (the importing module's __name__): a - # top-level name goes to workload_import_names if ANY non-roar module - # imported it, and to roar_import_names when a roar.* module did. Their - # difference (roar-only) is subtracted from the freeze so roar's own - # dependency footprint doesn't masquerade as the workload's. P0-11/P0-18. - self.workload_import_names: set[str] = set() - self.roar_import_names: set[str] = set() - # sys.modules keys present when install() runs — roar's own bootstrap - # footprint (roar + the deps it imports to build the tracker, before - # __import__ is patched, so tracking_import can't attribute them). Keys are - # snapshotted cheaply at install; their top-level names are resolved at - # write_log (off the hot start path). - self._install_baseline_keys: set[str] = set() if not hasattr(self._environ, _ORIGINAL_ENVIRON_GET_ATTR): with contextlib.suppress(Exception): @@ -407,10 +304,6 @@ def __init__( def install(self) -> None: """Patch builtins and environ access for activity capture.""" - # Everything loaded right now is roar's bootstrap footprint: the workload - # hasn't run yet. Snapshot the keys cheaply (a set copy) before patching - # __import__ so it covers deps roar imported to build the tracker itself. - self._install_baseline_keys = set(sys.modules) builtins.open = self.tracking_open builtins.__import__ = self.tracking_import setattr(self._environ, _ENVIRON_GET_METHOD_NAME, self.patched_environ_get) @@ -427,25 +320,6 @@ def tracking_open(self, *args, **kwargs): def tracking_import(self, name, globals=None, locals=None, fromlist=(), level=0): self.imported_modules.add(name) - # Attribute this import by its ORIGIN. A cache hit still calls __import__ - # with the requesting module's globals, so `import yaml` issued from the - # workload is observed here even when roar loaded yaml first at injection — - # which is exactly what a presence/timing check on sys.modules cannot see. - if level == 0 and name: - top = name.split(".")[0] - if top and not top.startswith("_") and top not in _STDLIB: - origin = (globals or {}).get("__name__") or "" - if origin == "roar" or origin.startswith("roar."): - self.roar_import_names.add(top) - else: - origin_file = (globals or {}).get("__file__") - if origin_file and _site_packages_top(origin_file) is None: - # Only the workload's OWN loose code — the entry script, - # local modules, an editable repo — protects a name from - # roar-dependency exclusion. An installed package's internal - # import (e.g. pydantic importing itself) lives under a - # package root, so it can't masquerade as a workload need. - self.workload_import_names.add(top) module = self._real_import(name, globals, locals, fromlist, level) if self._environ.get("ROAR_WRAP") != "1": @@ -485,42 +359,12 @@ def write_log(self) -> None: for name, module in sys.modules.items() if getattr(module, "__file__", None) } - # roar's declared dependency closure, mapped from distribution names to the - # import names the file pass keys on. Catches deps roar pulls in via - # importlib (e.g. the ABI gate importing pydantic) that neither the origin - # hook nor the install baseline can observe. - closure_dists = roar_dependency_closure() - try: - from importlib import metadata as importlib_metadata - - pkg_dist_map = importlib_metadata.packages_distributions() - except Exception: - pkg_dist_map = {} - closure_imports = { - imp - for imp, dists in pkg_dist_map.items() - if any(_normalize_dist(d) in closure_dists for d in dists) - } # pkg_dist_map is reused by get_used_packages below (built once). - # roar-only footprint = what roar imported (post-install, by origin), what - # was already loaded at install time (bootstrap), and roar's declared - # dependency closure — MINUS anything the workload itself imported. That - # last subtraction is what keeps a shared dependency (e.g. pyyaml on a timm - # row) in the freeze even though roar depends on and loaded it too. - # P0-11/P0-18. - install_baseline = _installed_top_names( - sys.modules[k] for k in self._install_baseline_keys if k in sys.modules - ) - roar_exclusive = ( - self.roar_import_names | install_baseline | closure_imports - ) - self.workload_import_names used_packages = get_used_packages( modules_files, installed_packages, sorted(self.imported_modules), workload_root=os.getcwd(), loaded_files=loaded_files, - roar_exclusive_names=sorted(roar_exclusive), - pkg_dist_map=pkg_dist_map, ) data = { "opened_files": sorted(self.opened_files), @@ -528,7 +372,6 @@ def write_log(self) -> None: "env_reads": dict(sorted(self.env_reads.items())), "modules_files": modules_files, "roar_inject_dir": self._inject_dir, - "roar_exclusive": sorted(roar_exclusive), "shared_libs": get_loaded_shared_libs(self._real_open), "sys_prefix": sys.prefix, "sys_base_prefix": sys.base_prefix, From 8b1a5c2aa9f322a596677dbc0878ed5bd9969dbd Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Mon, 10 Aug 2026 21:34:44 +0000 Subject: [PATCH 35/52] P0-11 (broad): subtract roar's footprint by LOCATION, not name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The campaign runs roar in its own uv-tool venv, ABI-matched to the workload (the mandatory P0-14 layout). In that configuration ROAR_RUNTIME_PYTHONPATH_ACTIVE is null, so the existing runtime-path filter is inert and roar's dependency footprint (blake3, click, cryptography, the pydantic tree, ...) leaks into the workload freeze — P0-11 broad, the 21/57 pins Carl measured on row 005. The rc3 attempt (reverted, P0-28) subtracted by distribution NAME and stripped the workload's own copies of tqdm / typing-extensions, because roar and the workload have same-named deps in different venvs. This fixes it by LOCATION: derive roar's install root structurally from the inject dir and exclude modules loaded from it. Path distinguishes roar's copy of typing_extensions from the workload's copy in a different venv; name cannot. - roar_footprint_paths(inject_dir, sys_prefix): roar's install root, but only when ISOLATED (root outside the interpreter prefix). When roar shares the workload venv, path can't tell the copies apart, so it returns nothing and the freeze safely OVER-includes rather than risk a false negative. - write_log folds it into the exclusion set used by both the modules_files filter and get_installed_packages, so it also arms when _ACTIVE is null. Validated end-to-end in the matched-ABI / _ACTIVE-null isolated cell that rc3's oracle missed (roar in its own tool tree, workload on a separate venv): the reverted build leaks blake3/click/cryptography; this build removes them while keeping the workload's typing_extensions (the P0-28 casualty). Unit tests in test_roar_footprint_location.py encode the name-fails / location-succeeds contrast and the shared-venv guard. Full runtime suite 123 passed. Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/execution/runtime/inject/tracker.py | 52 ++++++++++++- .../runtime/test_roar_footprint_location.py | 74 +++++++++++++++++++ 2 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 tests/execution/runtime/test_roar_footprint_location.py diff --git a/roar/execution/runtime/inject/tracker.py b/roar/execution/runtime/inject/tracker.py index 3e9dc38b..fe88cdc6 100644 --- a/roar/execution/runtime/inject/tracker.py +++ b/roar/execution/runtime/inject/tracker.py @@ -274,6 +274,43 @@ def is_under_any_runtime_path(path: str, runtime_paths: Sequence[str]) -> bool: return False +def _roar_site_packages_root(inject_dir: str) -> str | None: + """roar's own install root — the site-packages (or ``uv tool`` venv) holding + the roar package. ``inject_dir`` is ``/roar/execution/runtime/inject``, + so the root is four levels up. None if the shape is unexpected.""" + root = os.path.abspath(inject_dir) + for _ in range(4): + parent = os.path.dirname(root) + if parent == root: + return None + root = parent + return root + + +def roar_footprint_paths(inject_dir: str, sys_prefix: str) -> tuple[str, ...]: + """Location(s) whose loaded modules are roar's OWN footprint, to subtract from + the freeze by PATH — never by name. + + The campaign runs roar in its own ``uv tool`` venv, ABI-matched to the workload + (the mandatory P0-14 layout). There ``ROAR_RUNTIME_PYTHONPATH_ACTIVE`` is null, + so the runtime-path filter is inert and roar's dependency footprint leaks into + the freeze — P0-11 (broad). roar's install root is knowable structurally, so we + exclude modules loaded from it. This distinguishes roar's copy of a package + from a same-named copy in the workload's venv (a different path); name-keying + could not, and stripped the workload's own tqdm / typing-extensions — P0-28. + + Guarded to the ISOLATED case: when roar shares the workload venv (its root is + under the interpreter's own prefix), path cannot tell the copies apart, so this + returns nothing and the freeze safely OVER-includes rather than risk dropping a + workload dependency.""" + root = _roar_site_packages_root(inject_dir) + if not root: + return () + if is_under_any_runtime_path(root, (sys_prefix,)): + return () # shared venv — do not path-exclude (over-include is the safe side) + return (root,) + + class RuntimeInjectionTracker: """Capture generic process activity for the parent-side recorder.""" @@ -338,6 +375,14 @@ def write_log(self) -> None: return runtime_pythonpath = get_active_runtime_pythonpath(self._environ) + # Also exclude roar's own install root by LOCATION. In the campaign's + # ABI-matched uv-tool layout ROAR_RUNTIME_PYTHONPATH_ACTIVE is null, so the + # runtime-path filter alone leaves roar's dependency footprint in the freeze + # (P0-11 broad). Path-keyed, never name-keyed (P0-28), so a same-named + # workload copy in a different venv survives. + exclusion_paths = runtime_pythonpath + roar_footprint_paths( + self._inject_dir, sys.prefix + ) modules_files = sorted( os.path.abspath(getattr(module, "__file__", "")) for module in sys.modules.values() @@ -345,11 +390,12 @@ def write_log(self) -> None: and not os.path.abspath(getattr(module, "__file__", "")).startswith(self._inject_dir) and not is_under_any_runtime_path( os.path.abspath(getattr(module, "__file__", "")), - runtime_pythonpath, + exclusion_paths, ) ) - # Trev's #268: exclude roar's own runtime-tree dists from the installed set. - installed_packages = get_installed_packages(excluded_paths=runtime_pythonpath) + # Trev's #268 + P0-11 broad: exclude roar's runtime-tree AND install-root + # dists from the installed set, so the file pass can't resolve them. + installed_packages = get_installed_packages(excluded_paths=exclusion_paths) # name -> loaded module file, so get_used_packages can tell an ALIASED # import (sys.modules[name] resolves to a different package) from a normal # or merely-probed one. Keyed by the sys.modules key (the import name), diff --git a/tests/execution/runtime/test_roar_footprint_location.py b/tests/execution/runtime/test_roar_footprint_location.py new file mode 100644 index 00000000..f7a09e42 --- /dev/null +++ b/tests/execution/runtime/test_roar_footprint_location.py @@ -0,0 +1,74 @@ +"""P0-11 (broad) / P0-28: roar's own dependency footprint must be subtracted from +the freeze by the LOCATION it loaded from, never by package NAME. + +The campaign runs roar in its own ``uv tool`` venv, ABI-matched to the workload, +so roar's deps and the workload's deps live in two different venvs but share +distribution *names* (e.g. ``typing_extensions`` is a dep of both roar's pydantic +and the workload's torch). Name-keyed subtraction (the reverted rc3 attempt) +stripped the workload's own copy — P0-28. Location-keyed subtraction removes only +what actually loaded from roar's install root, so the workload's copy survives. +""" + +from __future__ import annotations + +from roar.execution.runtime.inject.tracker import ( + _site_packages_top, + get_used_packages, + is_under_any_runtime_path, + roar_footprint_paths, +) + + +def test_footprint_excluded_by_location_not_by_name(tmp_path): + """The regression test for P0-28: a workload dependency that shares a NAME with + a roar dependency survives, because we key on where it loaded from.""" + roar_root = tmp_path / "uv-tools" / "roar-cli" / "lib" / "python3.12" / "site-packages" + wl_root = tmp_path / "wlvenv" / "lib" / "python3.12" / "site-packages" + inject_dir = str(roar_root / "roar" / "execution" / "runtime" / "inject") + wl_prefix = str(tmp_path / "wlvenv") + + # Loaded modules: roar's OWN click + typing_extensions (from roar_root), and the + # WORKLOAD's OWN typing_extensions + torch (from wl_root). typing_extensions + # collides on name across the two venvs — the exact P0-28 case. + loaded = [ + str(roar_root / "click" / "__init__.py"), + str(roar_root / "typing_extensions.py"), + str(wl_root / "typing_extensions.py"), + str(wl_root / "torch" / "__init__.py"), + ] + installed = {"click": "8.1.0", "typing_extensions": "4.16.0", "torch": "2.7.0"} + roar_dep_names = {"click", "typing_extensions"} # both are roar deps, by name + + # (1) NAME-keying (rc3) would drop the workload's typing_extensions as well — + # the false negative P0-28 reported. + name_kept = [f for f in loaded if _site_packages_top(f) not in roar_dep_names] + assert not any("typing_extensions" in f for f in name_kept), ( + "name-keying strips the workload's own typing_extensions — this is the P0-28 bug" + ) + + # (2) LOCATION-keying (the fix): exclude only what loaded from roar's root. + excl = roar_footprint_paths(inject_dir, wl_prefix) + assert excl, "isolated roar install root must be excluded" + loc_kept = [f for f in loaded if not is_under_any_runtime_path(f, excl)] + used = get_used_packages(loc_kept, installed) + + # (3) the workload's typing_extensions and torch survive; roar's click is gone. + assert "typing_extensions" in used, "workload's own typing_extensions must survive" + assert "torch" in used + assert "click" not in used, "roar's footprint must be gone" + + +def test_shared_venv_is_not_location_excluded(tmp_path): + """When roar is pip-installed in the workload's own venv, its root is under the + interpreter prefix; path cannot tell the copies apart, so no location exclusion + is applied and the freeze safely over-includes (never a false negative).""" + venv = tmp_path / "venv" + inject_dir = str(venv / "lib" / "python3.12" / "site-packages" / "roar" / "execution" / "runtime" / "inject") + assert roar_footprint_paths(inject_dir, str(venv)) == () + + +def test_isolated_root_is_reported(tmp_path): + """The isolated ``uv tool`` root (outside the workload prefix) is returned.""" + roar_root = tmp_path / "uv-tools" / "roar-cli" / "lib" / "python3.12" / "site-packages" + inject_dir = str(roar_root / "roar" / "execution" / "runtime" / "inject") + assert roar_footprint_paths(inject_dir, str(tmp_path / "wlvenv")) == (str(roar_root),) From b54e298682573f4d0089ceb5f0f1dae979c47f32 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Mon, 10 Aug 2026 21:36:18 +0000 Subject: [PATCH 36/52] trackio shim: provide run.get_url() so wandb callers don't crash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wandb code (e.g. lerobot's logging) calls run.get_url(); trackio's Run has no such method, so the aliased wandb -> trackio path raises. Add get_url() in the same compat block that already backfills run.summary. Compose the HF Spaces URL from space_id + project rather than aliasing run.url: run.url exists but returns the bare space id, not a URL, so aliasing it would stop the crash while publishing a broken link that still passes a smoke test — exactly the 404-shaped artefact to avoid. Both URL components are already in scope in the init wrapper. Unblocks 007's re-capture with --wandb.enable=true (recipe-side, no fork change). The 200-resolve verification is done separately against a live shim call. Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/integrations/wandb_trackio.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/roar/integrations/wandb_trackio.py b/roar/integrations/wandb_trackio.py index 7688af18..0de40240 100644 --- a/roar/integrations/wandb_trackio.py +++ b/roar/integrations/wandb_trackio.py @@ -129,6 +129,14 @@ def init(*args, **kwargs): try: if not hasattr(run, "summary"): run.summary = {} + if not hasattr(run, "get_url"): + # wandb code (e.g. lerobot) calls run.get_url(); trackio's Run has + # no such method. run.url exists but returns the bare space id, not + # a URL — aliasing it would stop the crash and publish a broken link + # that still passes a smoke test. COMPOSE the Spaces URL instead; + # space_id and project are both in scope here. + _project = kwargs.get("project") + run.get_url = lambda: f"https://huggingface.co/spaces/{space_id}?project={_project}" except Exception: pass trackio.run = run From 80d797d579d682cd3b10cd37abea0c277361d801 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Tue, 11 Aug 2026 00:15:46 +0000 Subject: [PATCH 37/52] trackio shim: adapt lerobot's wandb calls (resume=None, log(data=)) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more caller-assumption gaps the 007 (lerobot) capture hit, both in the existing init()/log() wrappers: - init(): wandb's `resume` default is None ("do not resume"); trackio accepts only must/allow/never and raises ValueError on None. lerobot always passes the kwarg (`resume="must" if cfg.resume else None`), so it died in init(). Drop a None resume before trackio sees it. - log(): wandb's first parameter is named `data`; trackio names it `metrics`, so `wandb.log(data=..., step=...)` is a TypeError. Forward `data=` positionally. With #279's get_url(), these were the three blockers between lerobot and the trackio shim. Verified against trackio 0.34.0 and installed rc4 (Carl), and here with regression tests asserting the metrics are actually forwarded — not merely that no exception was raised (the HTTP-200 gate is worthless: a Gradio Space returns 200 for any project, even a nonexistent one). Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/integrations/wandb_trackio.py | 11 ++++++++ tests/integrations/test_wandb_trackio.py | 36 ++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/roar/integrations/wandb_trackio.py b/roar/integrations/wandb_trackio.py index 0de40240..b2420d2b 100644 --- a/roar/integrations/wandb_trackio.py +++ b/roar/integrations/wandb_trackio.py @@ -124,6 +124,12 @@ def _install_trackio_alias(space_id: str) -> bool: def init(*args, **kwargs): for k in _WANDB_ONLY_INIT: kwargs.pop(k, None) + # wandb's `resume` default is None ("do not resume"); trackio accepts only + # "must"/"allow"/"never" and RAISES ValueError on None. Callers that always + # pass the kwarg (lerobot: `resume="must" if cfg.resume else None`) therefore + # die in init(). Map wandb's None onto trackio's "never" — same meaning. + if kwargs.get("resume") is None: + kwargs.pop("resume", None) kwargs.setdefault("space_id", space_id) run = _orig_init(*args, **kwargs) try: @@ -148,6 +154,11 @@ def init(*args, **kwargs): def log(*args, **kwargs): kwargs.pop("commit", None) + # wandb's first parameter is NAMED `data`; trackio names it `metrics`. Callers + # using the keyword form (lerobot: `wandb.log(data=batch_data, step=step)`) + # otherwise get TypeError: log() got an unexpected keyword argument 'data'. + if "data" in kwargs and not args: + args = (kwargs.pop("data"),) if args and isinstance(args[0], dict): args = (_to_jsonable(args[0]), *args[1:]) try: diff --git a/tests/integrations/test_wandb_trackio.py b/tests/integrations/test_wandb_trackio.py index 5e74d111..9a6d21b7 100644 --- a/tests/integrations/test_wandb_trackio.py +++ b/tests/integrations/test_wandb_trackio.py @@ -70,6 +70,42 @@ def test_sync_aliases_to_trackio_and_strips_wandb_only_kwargs(): assert "commit" not in calls["log"][1] # wandb-only log kwarg stripped +def test_lerobot_resume_none_is_dropped_before_trackio(): + """trackio.init raises ValueError on ``resume=None``; wandb's default IS None + and lerobot always passes the kwarg (``resume="must" if cfg.resume else None``), + so it dies in init(). The shim must drop a None resume.""" + calls: dict = {} + fake = types.ModuleType("trackio") + fake.init = lambda *a, **k: calls.__setitem__("init", k) or types.SimpleNamespace(summary={}) + fake.log = lambda *a, **k: None + sys.modules["trackio"] = fake + + wandb_trackio.install(environ={"ROAR_WANDB_TO_TRACKIO": "1", "TRACKIO_SPACE_ID": "org/space"}) + import wandb + + wandb.init(project="p", resume=None) # would raise inside trackio without the drop + assert "resume" not in calls["init"] + + +def test_lerobot_log_data_kwarg_is_forwarded_positionally(): + """``wandb.log(data=..., step=...)`` — wandb's first param is named ``data``, + trackio's is ``metrics``, so the keyword form is a TypeError. The shim forwards + it positionally so the metrics actually reach trackio.""" + calls: dict = {} + fake = types.ModuleType("trackio") + fake.init = lambda *a, **k: types.SimpleNamespace(summary={}) + fake.log = lambda *a, **k: calls.__setitem__("log", (a, k)) + sys.modules["trackio"] = fake + + wandb_trackio.install(environ={"ROAR_WANDB_TO_TRACKIO": "1", "TRACKIO_SPACE_ID": "org/space"}) + import wandb + + wandb.log(data={"loss": 0.5}, step=3) + args, kwargs = calls["log"] + assert args and args[0] == {"loss": 0.5} # metrics forwarded positionally + assert "data" not in kwargs # no TypeError-inducing keyword survives + + def test_off_beats_a_configured_space(): fake = types.ModuleType("trackio") fake.init = lambda *a, **k: None From c8209a1fb22ebae255e69e43cc62f132a2ce6004 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Tue, 11 Aug 2026 13:52:15 +0000 Subject: [PATCH 38/52] chore(lint): ruff format the #278 files (unbreak CI lint on rc/0.4.4) #278 merged with a ruff-format violation (ruff check passed but ruff format --check did not), failing the CI lint job for every branch off rc/0.4.4. Pure formatting; no behavior change. Verified: location + used_packages tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/execution/runtime/inject/tracker.py | 4 +--- tests/execution/runtime/test_roar_footprint_location.py | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roar/execution/runtime/inject/tracker.py b/roar/execution/runtime/inject/tracker.py index fe88cdc6..832247c5 100644 --- a/roar/execution/runtime/inject/tracker.py +++ b/roar/execution/runtime/inject/tracker.py @@ -380,9 +380,7 @@ def write_log(self) -> None: # runtime-path filter alone leaves roar's dependency footprint in the freeze # (P0-11 broad). Path-keyed, never name-keyed (P0-28), so a same-named # workload copy in a different venv survives. - exclusion_paths = runtime_pythonpath + roar_footprint_paths( - self._inject_dir, sys.prefix - ) + exclusion_paths = runtime_pythonpath + roar_footprint_paths(self._inject_dir, sys.prefix) modules_files = sorted( os.path.abspath(getattr(module, "__file__", "")) for module in sys.modules.values() diff --git a/tests/execution/runtime/test_roar_footprint_location.py b/tests/execution/runtime/test_roar_footprint_location.py index f7a09e42..0f5d6163 100644 --- a/tests/execution/runtime/test_roar_footprint_location.py +++ b/tests/execution/runtime/test_roar_footprint_location.py @@ -63,7 +63,9 @@ def test_shared_venv_is_not_location_excluded(tmp_path): interpreter prefix; path cannot tell the copies apart, so no location exclusion is applied and the freeze safely over-includes (never a false negative).""" venv = tmp_path / "venv" - inject_dir = str(venv / "lib" / "python3.12" / "site-packages" / "roar" / "execution" / "runtime" / "inject") + inject_dir = str( + venv / "lib" / "python3.12" / "site-packages" / "roar" / "execution" / "runtime" / "inject" + ) assert roar_footprint_paths(inject_dir, str(venv)) == () From 00040e57725ebe5b6ad30f945a496b5df94b4e08 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Tue, 11 Aug 2026 14:36:24 +0000 Subject: [PATCH 39/52] P0-22 (Option B): dedup job edges by content hash to match glaas (W2, reopened) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit glaas keys job inputs/outputs on (job_id, artifact_hash) and stages with skipDuplicates, so byte-identical outputs written to several names (timm os.link last/best/checkpoint = one inode, three names) collapse to ONE stored edge. roar asserted the raw per-path count at finalize, so those rows 400'd with "Staged lineage counts did not match" after a full paid GPU run — and roar's canonical session hash (per-path) would diverge from glaas's (per-content) for the same reason. Align roar to glaas's content-addressed model, no schema migration: - build_staged_lineage_counts counts DISTINCT content hashes per job. - build_canonical_session_payload dedups edges by hash (keeping the smallest path deterministically) so roar's session hash matches glaas's. This function is used by BOTH publish and reproduce lookup, so both stay consistent with each other and now with glaas. Both are a no-op for every currently-passing row (no duplicate-content edges -> distinct == path), so existing session hashes are unchanged — verified: the canonical-hash unit tests pass untouched. Also fixes the integration fake (fake_glaas) to model glaas's (job_id, artifact_hash) dedup in its staged-count and canonical-hash paths, so it no longer asserts a per-path count the real server never has. This is why my first attempt (#281) broke CI: the fake, not glaas, was the per-path contract. Why Option B over a glaas PK migration: glaas has never recorded duplicate paths (the PK has always dropped them), and reproduction/AI-BOM key on content + DAG structure, not filesystem-name multiplicity. So this loses nothing vs today and avoids a production PK migration + reproduce rework. Scope note (draft): the finalize-count fix is fully safe and is the P0-22 unblock. End-to-end hash agreement on a real duplicate-byte row also depends on glaas's skipDuplicates keeping the same representative path roar picks (smallest) — i.e. staging in sorted order — to be confirmed on a real dup-byte capture (nerf/012). compute_canonical_jobs_session_hash is intentionally left unchanged (no callers). Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/application/publish/session.py | 38 +++++++++++-- .../publish/test_staged_lineage_counts.py | 57 +++++++++++++++++++ tests/integration/fake_glaas.py | 26 +++++++-- 3 files changed, 110 insertions(+), 11 deletions(-) create mode 100644 tests/application/publish/test_staged_lineage_counts.py diff --git a/roar/application/publish/session.py b/roar/application/publish/session.py index 1ee28699..08a6fdde 100644 --- a/roar/application/publish/session.py +++ b/roar/application/publish/session.py @@ -45,6 +45,26 @@ class PreparedPublishSession: registration_session_mode: str | None = None +def _dedup_edges_by_hash(edges: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Collapse edges to one per CONTENT hash — matching glaas, which keys job + inputs/outputs on ``(job_id, artifact_hash)`` and so drops duplicate-path edges + for the same bytes (timm's ``os.link`` last/best/checkpoint = one inode, three + names). Keeps the lexicographically-smallest path so the surviving edge is + deterministic across roar and glaas. Edges without a resolvable hash are + dropped. A no-op when a job has no byte-identical edges — i.e. every currently + passing row — so existing session hashes are unchanged. P0-22. + """ + by_hash: dict[str, dict[str, Any]] = {} + for edge in edges: + digest = _canonical_artifact_hash(edge) + if not digest: + continue + current = by_hash.get(digest) + if current is None or str(edge.get("path") or "") < str(current.get("path") or ""): + by_hash[digest] = edge + return list(by_hash.values()) + + def build_canonical_session_payload( *, lineage: LineageData, @@ -63,7 +83,7 @@ def build_canonical_session_payload( "hash": _canonical_artifact_hash(artifact), "path": artifact.get("path"), } - for artifact in job.get("_inputs", []) + for artifact in _dedup_edges_by_hash(job.get("_inputs", [])) ], key=lambda artifact: ( str(artifact.get("hash") or ""), @@ -76,7 +96,7 @@ def build_canonical_session_payload( "hash": _canonical_artifact_hash(artifact), "path": artifact.get("path"), } - for artifact in job.get("_outputs", []) + for artifact in _dedup_edges_by_hash(job.get("_outputs", [])) ], key=lambda artifact: ( str(artifact.get("hash") or ""), @@ -110,11 +130,19 @@ def build_git_context_from_lineage(lineage: LineageData) -> GitContext: def build_staged_lineage_counts(jobs: list[dict[str, Any]]) -> dict[str, int]: - """Build lightweight finalize expectations for staged registration-session lineage.""" + """Finalize expectations for staged registration-session lineage. + + Counts DISTINCT content hashes per job — matching glaas, which stores job + edges keyed on ``(job_id, artifact_hash)`` and so collapses byte-identical + outputs written to several names (timm ``os.link`` last/best/checkpoint) to one + row. Asserting the raw per-path count made finalize 400 with "Staged lineage + counts did not match" (P0-22). A no-op for every currently-passing row (no + duplicate edges → distinct == path). + """ return { "jobs": len(jobs), - "inputs": sum(len(job.get("_inputs", [])) for job in jobs), - "outputs": sum(len(job.get("_outputs", [])) for job in jobs), + "inputs": sum(len(_dedup_edges_by_hash(job.get("_inputs", []))) for job in jobs), + "outputs": sum(len(_dedup_edges_by_hash(job.get("_outputs", []))) for job in jobs), } diff --git a/tests/application/publish/test_staged_lineage_counts.py b/tests/application/publish/test_staged_lineage_counts.py new file mode 100644 index 00000000..798e508a --- /dev/null +++ b/tests/application/publish/test_staged_lineage_counts.py @@ -0,0 +1,57 @@ +"""P0-22 (Option B): roar's finalize count AND canonical session hash must dedup +job edges by CONTENT hash, matching glaas's (job_id, artifact_hash) storage key — +so a workload writing identical bytes to two names (timm os.link last/best/ +checkpoint) doesn't 400 finalize or diverge from glaas's published hash. +""" + +from __future__ import annotations + +from roar.application.publish.session import ( + _dedup_edges_by_hash, + build_staged_lineage_counts, +) + + +def _job(inputs=(), outputs=()): + return {"_inputs": list(inputs), "_outputs": list(outputs)} + + +def test_hardlink_duplicate_outputs_collapse_to_one_content(): + dup = [ + {"hash": "abc", "path": "last.pth.tar", "byte_ranges": None}, + {"hash": "abc", "path": "checkpoint-9.pth.tar", "byte_ranges": None}, + {"hash": "abc", "path": "model_best.pth.tar", "byte_ranges": None}, + {"hash": "def", "path": "config.json", "byte_ranges": None}, + ] + assert build_staged_lineage_counts([_job(outputs=dup)])["outputs"] == 2 + + +def test_no_duplicates_is_a_noop_equal_to_path_count(): + outs = [{"hash": h, "path": h, "byte_ranges": None} for h in ("a", "b", "c")] + assert build_staged_lineage_counts([_job(outputs=outs)])["outputs"] == 3 + + +def test_dedup_keeps_smallest_path_deterministically(): + # matches glaas skipDuplicates when staged in sorted order; stable representative + edges = [ + {"hash": "x", "path": "zzz"}, + {"hash": "x", "path": "aaa"}, + {"hash": "x", "path": "mmm"}, + ] + kept = _dedup_edges_by_hash(edges) + assert len(kept) == 1 and kept[0]["path"] == "aaa" + + +def test_dedup_is_by_hash_not_byte_ranges(): + # glaas's key ignores byte_ranges, so same hash + different ranges still collapses + edges = [ + {"hash": "z", "path": "d", "byte_ranges": [[0, 100]]}, + {"hash": "z", "path": "d", "byte_ranges": [[100, 200]]}, + ] + assert len(_dedup_edges_by_hash(edges)) == 1 + + +def test_edges_without_a_hash_are_dropped(): + assert ( + build_staged_lineage_counts([_job(outputs=[{"path": "x"}, {"hash": ""}])])["outputs"] == 0 + ) diff --git a/tests/integration/fake_glaas.py b/tests/integration/fake_glaas.py index 4e7dd90c..5ba98e17 100644 --- a/tests/integration/fake_glaas.py +++ b/tests/integration/fake_glaas.py @@ -202,8 +202,8 @@ def _count_registration_session_staging( jobs = [job for job in jobs_by_uid.values() if isinstance(job, dict)] return { "jobs": len(jobs), - "inputs": sum(len(job.get("inputs", [])) for job in jobs), - "outputs": sum(len(job.get("outputs", [])) for job in jobs), + "inputs": sum(len(_dedup_artifacts_by_hash(job.get("inputs", []))) for job in jobs), + "outputs": sum(len(_dedup_artifacts_by_hash(job.get("outputs", []))) for job in jobs), } def _compute_registration_session_hash( @@ -230,8 +230,7 @@ def _compute_registration_session_hash( inputs = sorted( [ {"hash": artifact.get("hash"), "path": artifact.get("path")} - for artifact in job.get("inputs", []) - if isinstance(artifact, dict) + for artifact in _dedup_artifacts_by_hash(job.get("inputs", [])) ], key=lambda artifact: ( str(artifact.get("hash") or ""), @@ -241,8 +240,7 @@ def _compute_registration_session_hash( outputs = sorted( [ {"hash": artifact.get("hash"), "path": artifact.get("path")} - for artifact in job.get("outputs", []) - if isinstance(artifact, dict) + for artifact in _dedup_artifacts_by_hash(job.get("outputs", [])) ], key=lambda artifact: ( str(artifact.get("hash") or ""), @@ -1154,6 +1152,22 @@ def log_message(self, format: str, *args: object) -> None: """Suppress default stderr logging for integration tests.""" +def _dedup_artifacts_by_hash(artifacts: Any) -> list[dict[str, Any]]: + """Model glaas's ``(job_id, artifact_hash)`` storage key: byte-identical edges + written to several paths collapse to one stored row, keeping the + lexicographically-smallest path. Mirrors roar's staged-count / canonical dedup + so this fake counts and hashes the way the real server stores (P0-22).""" + by_hash: dict[str, dict[str, Any]] = {} + for artifact in artifacts if isinstance(artifacts, list) else []: + if not isinstance(artifact, dict) or not artifact.get("hash"): + continue + digest = artifact["hash"] + current = by_hash.get(digest) + if current is None or str(artifact.get("path") or "") < str(current.get("path") or ""): + by_hash[digest] = artifact + return list(by_hash.values()) + + def _parse_metadata_object(value: Any) -> dict[str, Any]: if isinstance(value, dict): return value From 9d2f5d201e635698826aa35a1da26549215afcc2 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Tue, 18 Aug 2026 19:17:51 +0000 Subject: [PATCH 40/52] fix: record the command the user ran, not the one /proc reports The root process's argv was read back from /proc by every tracer. /proc reports what the kernel ran rather than what was asked for, so the two differ in ways that matter: roar run ./train.sh #!/usr/bin/env python3 recorded: ['/usr/bin/env', 'python3', './train.sh'] That value is not incidental -- runtime_collector feeds the root process's argv to RuntimeInfo.command, so it is the run's recorded command in provenance. Recording the kernel's rewrite instead of the user's command is wrong for a tool whose job is to say what ran. It is also nondeterministic. A process that exits before the read is a zombie, whose /proc entry survives while its memory is torn down, so cmdline reads back empty. The same run then records two different commands depending on machine load. roar launches the workload, so the root's argv is known exactly and needs no discovery. Prefer it in all three tracers. Descendants have no such source and keep using /proc, which is also what build_pip_collector needs. The eBPF path had no way to know it: Register now carries root_command. The field is #[serde(default)] and the wire format is field-named MessagePack, so a long-lived roard and a client of a different version still interoperate -- an older daemon ignores the extra key, and an older client's message decodes as empty, which every caller reads as "fall back to /proc". Co-Authored-By: Claude Opus 5 (1M context) --- rust/tracers/ebpf/userspace/src/daemon.rs | 65 +++++++++----- rust/tracers/ebpf/userspace/src/ipc.rs | 53 +++++++++++- rust/tracers/ebpf/userspace/src/main.rs | 1 + rust/tracers/preload/src/main.rs | 65 +++++++++++--- rust/tracers/ptrace/src/main.rs | 34 +++++++- .../test_recorded_command_is_what_was_run.py | 86 +++++++++++++++++++ 6 files changed, 264 insertions(+), 40 deletions(-) create mode 100644 tests/integration/test_recorded_command_is_what_was_run.py diff --git a/rust/tracers/ebpf/userspace/src/daemon.rs b/rust/tracers/ebpf/userspace/src/daemon.rs index b38a4a4c..a372e6fd 100644 --- a/rust/tracers/ebpf/userspace/src/daemon.rs +++ b/rust/tracers/ebpf/userspace/src/daemon.rs @@ -13,7 +13,7 @@ use tracer_runtime::timestamp_now; use crate::events; use crate::ipc::{self, ClientMessage, DaemonMessage}; -use crate::state::{TracerOutput, TracerState}; +use crate::state::{ProcessInfo, TracerOutput, TracerState}; // ── State types ────────────────────────────────────────────────────────────── @@ -46,13 +46,31 @@ impl DaemonState { } } - pub fn register(&mut self, run_id: u64, root_pid: u32) { + pub fn register(&mut self, run_id: u64, root_pid: u32, root_command: Vec) { let mut tracer = TracerState::new(None); tracer.start_time = timestamp_now(); tracer.active_pids.insert(root_pid); // Capture initial process info from /proc (child is SIGSTOP'd but exists) - if let Some(info) = crate::state::capture_process_info(root_pid, None) { + let mut info = crate::state::capture_process_info(root_pid, None); + if !root_command.is_empty() { + // The client told us what it was asked to run, which is + // authoritative: /proc reports the post-exec argv, so a + // `#!/usr/bin/env python3` script reads back as + // `/usr/bin/env python3 ./train.sh` rather than `./train.sh`. + match info.as_mut() { + Some(info) => info.command = root_command, + None => { + info = Some(ProcessInfo { + pid: root_pid, + parent_pid: None, + command: root_command, + env: HashMap::new(), + }) + } + } + } + if let Some(info) = info { tracer.processes.insert(root_pid, info); } @@ -366,9 +384,16 @@ fn handle_client( }; match msg { - ClientMessage::Register { run_id, root_pid } => { + ClientMessage::Register { + run_id, + root_pid, + root_command, + } => { info!("register: run_id={run_id} pid={root_pid}"); - state.lock().unwrap().register(run_id, root_pid); + state + .lock() + .unwrap() + .register(run_id, root_pid, root_command); ipc::send_message(&mut stream, &DaemonMessage::Ack { run_id })?; } ClientMessage::Deregister { run_id } => { @@ -419,7 +444,7 @@ mod tests { #[test] fn test_register_creates_run_state() { let mut state = DaemonState::new(); - state.register(1, 100); + state.register(1, 100, vec![]); assert!(state.runs.contains_key(&1)); let run = &state.runs[&1]; @@ -433,8 +458,8 @@ mod tests { #[test] fn test_deregister_marks_completed_and_keeps_pid_to_run() { let mut state = DaemonState::new(); - state.register(1, 100); - state.register(2, 200); + state.register(1, 100, vec![]); + state.register(2, 200, vec![]); let remaining = state.deregister(1); assert_eq!(remaining, 1); @@ -450,8 +475,8 @@ mod tests { #[test] fn test_get_report_clears_pid_to_run_for_run() { let mut state = DaemonState::new(); - state.register(1, 100); - state.register(2, 200); + state.register(1, 100, vec![]); + state.register(2, 200, vec![]); state.deregister(1); // Still routable until get_report. @@ -466,7 +491,7 @@ mod tests { #[test] fn test_get_report_returns_real_data() { let mut state = DaemonState::new(); - state.register(1, 100); + state.register(1, 100, vec![]); // Simulate some file I/O via the TracerState state @@ -539,7 +564,7 @@ mod tests { #[test] fn test_late_event_after_deregister_is_still_routed() { let mut state = DaemonState::new(); - state.register(1, 100); + state.register(1, 100, vec![]); // Register an open so the FD tracker has a path mapping. state @@ -590,7 +615,7 @@ mod tests { #[test] fn test_event_after_get_report_is_dropped_safely() { let mut state = DaemonState::new(); - state.register(1, 100); + state.register(1, 100, vec![]); state .runs .get_mut(&1) @@ -611,8 +636,8 @@ mod tests { let mut state = DaemonState::new(); assert_eq!(state.active_run_count(), 0); - state.register(1, 100); - state.register(2, 200); + state.register(1, 100, vec![]); + state.register(2, 200, vec![]); assert_eq!(state.active_run_count(), 2); state.deregister(1); @@ -625,9 +650,9 @@ mod tests { #[test] fn test_multiple_registrations_independent() { let mut state = DaemonState::new(); - state.register(1, 100); - state.register(2, 200); - state.register(3, 300); + state.register(1, 100, vec![]); + state.register(2, 200, vec![]); + state.register(3, 300, vec![]); assert_eq!(state.active_run_count(), 3); assert_eq!(state.pid_to_run.get(&100), Some(&1)); @@ -648,8 +673,8 @@ mod tests { #[test] fn test_process_event_routes_to_correct_run() { let mut state = DaemonState::new(); - state.register(1, 100); - state.register(2, 200); + state.register(1, 100, vec![]); + state.register(2, 200, vec![]); // Open a file on run 1's PID (pid=100) state diff --git a/rust/tracers/ebpf/userspace/src/ipc.rs b/rust/tracers/ebpf/userspace/src/ipc.rs index 7d645b46..1e6f696f 100644 --- a/rust/tracers/ebpf/userspace/src/ipc.rs +++ b/rust/tracers/ebpf/userspace/src/ipc.rs @@ -15,9 +15,23 @@ const MAX_PAYLOAD_SIZE: u32 = 16 * 1024 * 1024; #[derive(Serialize, Deserialize, Debug, PartialEq)] #[serde(tag = "type")] pub enum ClientMessage { - Register { run_id: u64, root_pid: u32 }, - Deregister { run_id: u64 }, - GetReport { run_id: u64 }, + Register { + run_id: u64, + root_pid: u32, + /// The command the client was asked to run. Authoritative for the root + /// process, which /proc reports post-exec. Defaulted so a client and + /// daemon of different versions still speak to each other -- the wire + /// format is field-named MessagePack, so the extra key is ignored by an + /// older daemon and absent-means-empty for an older client. + #[serde(default)] + root_command: Vec, + }, + Deregister { + run_id: u64, + }, + GetReport { + run_id: u64, + }, Ping, } @@ -92,6 +106,7 @@ mod tests { ClientMessage::Register { run_id: 42, root_pid: 1234, + root_command: vec!["./train.sh".to_string()], }, ClientMessage::Deregister { run_id: 42 }, ClientMessage::GetReport { run_id: 42 }, @@ -146,6 +161,7 @@ mod tests { let msg = ClientMessage::Register { run_id: 99, root_pid: 5678, + root_command: vec!["./train.sh".to_string()], }; let payload = rmp_serde::to_vec_named(&msg).unwrap(); @@ -167,6 +183,7 @@ mod tests { let msg = ClientMessage::Register { run_id: 123, root_pid: 4567, + root_command: vec!["python".to_string(), "train.py".to_string()], }; send_message(&mut a, &msg).unwrap(); @@ -174,6 +191,36 @@ mod tests { assert_eq!(received, msg); } + /// `roard` is long-lived, so a running daemon can predate the client that + /// connects to it (and vice versa across an upgrade). The wire format is + /// field-named MessagePack, so a Register that omits root_command must + /// still decode -- as empty, which every caller treats as "fall back to + /// /proc" rather than as an empty command line. + #[test] + fn a_register_without_root_command_still_decodes() { + #[derive(Serialize)] + #[serde(tag = "type")] + enum LegacyClientMessage { + Register { run_id: u64, root_pid: u32 }, + } + + let legacy = LegacyClientMessage::Register { + run_id: 7, + root_pid: 4242, + }; + let payload = rmp_serde::to_vec_named(&legacy).unwrap(); + + let decoded: ClientMessage = rmp_serde::from_slice(&payload).unwrap(); + assert_eq!( + decoded, + ClientMessage::Register { + run_id: 7, + root_pid: 4242, + root_command: vec![], + } + ); + } + #[test] fn test_socket_path_format() { let path = socket_path(); diff --git a/rust/tracers/ebpf/userspace/src/main.rs b/rust/tracers/ebpf/userspace/src/main.rs index 30b06aea..1c17db36 100644 --- a/rust/tracers/ebpf/userspace/src/main.rs +++ b/rust/tracers/ebpf/userspace/src/main.rs @@ -201,6 +201,7 @@ fn try_daemon_mode(output_file: &str, command: &[String]) -> Result { &ipc::ClientMessage::Register { run_id, root_pid: child_pid, + root_command: command.to_vec(), }, )?; diff --git a/rust/tracers/preload/src/main.rs b/rust/tracers/preload/src/main.rs index d05fce38..8756264e 100644 --- a/rust/tracers/preload/src/main.rs +++ b/rust/tracers/preload/src/main.rs @@ -398,21 +398,26 @@ impl CollectorState { } else { parent_pid }; - let info = capture_process_info(pid, fallback_parent).unwrap_or_else(|| ProcessInfo { + let mut info = capture_process_info(pid, fallback_parent).unwrap_or_else(|| ProcessInfo { pid, parent_pid: fallback_parent, - command: if pid == self.root_pid { - self.root_command.clone() - } else { - Vec::new() - }, - env: if pid == self.root_pid { - self.root_env.clone() - } else { - HashMap::new() - }, + command: Vec::new(), + env: HashMap::new(), }); + if pid == self.root_pid { + // We launched this process, so its argv is known exactly. Prefer it + // over /proc, which reports what the kernel ran rather than what the + // user asked for: a `#!/usr/bin/env python3` script reads back as + // `/usr/bin/env python3 ./train.sh`, and is empty altogether if the + // process exits before the read. Descendants have no such source and + // keep using /proc. + info.command = self.root_command.clone(); + if info.env.is_empty() { + info.env = self.root_env.clone(); + } + } + self.processes.insert(pid, info); } @@ -1030,6 +1035,34 @@ mod tests { use super::*; use tracer_schema::FileRecord; + /// The root's argv is what we were asked to run, not what /proc reports. + /// Using our own live pid as the root makes the two differ observably: the + /// launcher command below is nothing like this test binary's real argv. + #[test] + fn the_root_command_comes_from_the_launcher_not_proc() { + let launched = vec!["./train.sh".to_string()]; + let mut state = CollectorState::new(std::process::id(), launched.clone()); + + state.ensure_process(std::process::id()); + + let root = state.processes.get(&std::process::id()).unwrap(); + assert_eq!(root.command, launched); + // /proc was still consulted for everything else. + assert!(!root.env.is_empty(), "env should still come from /proc"); + } + + /// Descendants have no launcher-supplied argv, so they keep using /proc. + #[test] + fn a_descendant_command_still_comes_from_proc() { + let mut state = CollectorState::new(1, vec!["./train.sh".to_string()]); + + state.ensure_process(std::process::id()); + + let child = state.processes.get(&std::process::id()).unwrap(); + assert_ne!(child.command, vec!["./train.sh".to_string()]); + assert!(!child.command.is_empty()); + } + fn written_record(path: &str) -> FileRecord { FileRecord { path: path.to_string(), @@ -1076,7 +1109,10 @@ mod tests { state.reconcile_renamed_outputs(&mut summary); - assert_eq!(summary.files[0].path, final_str, "record rewritten to final name"); + assert_eq!( + summary.files[0].path, final_str, + "record rewritten to final name" + ); assert!(summary.written_files.contains(&final_str)); assert!(!summary.written_files.contains(&temp_str)); let _ = fs::remove_dir_all(&dir); @@ -1108,7 +1144,10 @@ mod tests { }; state.reconcile_renamed_outputs(&mut summary); - assert_eq!(summary.files[0].path, temp_str, "deleted file path unchanged"); + assert_eq!( + summary.files[0].path, temp_str, + "deleted file path unchanged" + ); let _ = fs::remove_dir_all(&dir); } diff --git a/rust/tracers/ptrace/src/main.rs b/rust/tracers/ptrace/src/main.rs index 0185acee..2875e300 100644 --- a/rust/tracers/ptrace/src/main.rs +++ b/rust/tracers/ptrace/src/main.rs @@ -108,10 +108,16 @@ struct TracerState { // CWD cache per PID cwd_cache: HashMap, + + // The command the launcher was asked to run, and the pid it became. This + // is authoritative for the root process: /proc reports what the kernel ran + // rather than what the user asked for. + root_pid: Option, + root_command: Vec, } impl TracerState { - fn new() -> Self { + fn new(root_command: Vec) -> Self { TracerState { processes: HashMap::new(), fd_tracker: FdTracker::new(None), @@ -126,6 +132,8 @@ impl TracerState { pending_fchdirs: HashMap::new(), active_pids: HashSet::new(), cwd_cache: HashMap::new(), + root_pid: None, + root_command, } } } @@ -170,9 +178,26 @@ fn capture_process_info(pid: Pid, state: &mut TracerState, parent_pid: Option info, + // The root's argv came from the launcher, so it is worth recording even + // when /proc could not be read at all. + None if is_root => ProcessInfo { + pid: pid_raw as u32, + parent_pid, + command: Vec::new(), + env: HashMap::new(), + }, + None => return, + }; + if is_root && !state.root_command.is_empty() { + // Authoritative: we launched it. /proc would report the post-exec argv, + // so a `#!/usr/bin/env python3` script reads back as + // `/usr/bin/env python3 ./train.sh` rather than `./train.sh`. + info.command = state.root_command.clone(); } + state.processes.insert(pid_raw, info); } // ============================================================================= @@ -781,7 +806,7 @@ fn run_preflight(json_output: bool, command: Option<&str>) -> i32 { fn run_tracer(command: Vec, output_file: &str) -> i32 { let start_time = timestamp_now(); - let mut state = TracerState::new(); + let mut state = TracerState::new(command.clone()); // Fork and trace match unsafe { fork() } { @@ -807,6 +832,7 @@ fn run_tracer(command: Vec, output_file: &str) -> i32 { // Parent: wait for child to stop at exec, then trace let child_pid = child.as_raw(); state.active_pids.insert(child_pid); + state.root_pid = Some(child_pid); // Wait for initial stop match waitpid(child, None) { diff --git a/tests/integration/test_recorded_command_is_what_was_run.py b/tests/integration/test_recorded_command_is_what_was_run.py new file mode 100644 index 00000000..8a71dad5 --- /dev/null +++ b/tests/integration/test_recorded_command_is_what_was_run.py @@ -0,0 +1,86 @@ +"""Tests that a run records the command the user asked for. + +The root process's argv used to be read back from /proc, which reports what the +kernel ran rather than what was requested. A `#!/usr/bin/env python3` script +therefore recorded as `/usr/bin/env python3 ./train.sh`, and a process that +exited before the read recorded nothing at all -- so the same run could be +recorded two different ways depending on machine load. + +roar launches the workload, so its argv is known exactly. Descendants have no +such source and still come from /proc. +""" + +from __future__ import annotations + +import json +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.integration + + +def _roar(cwd: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-m", "roar", *args], + cwd=cwd, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + + +def _latest_command(cwd: Path) -> list[str]: + connection = sqlite3.connect(cwd / ".roar" / "roar.db") + row = connection.execute("SELECT metadata FROM jobs ORDER BY id DESC LIMIT 1").fetchone() + connection.close() + assert row is not None + return (json.loads(row[0]).get("runtime") or {}).get("command") + + +@pytest.fixture +def initialised(tmp_path: Path) -> Path: + assert _roar(tmp_path, "init", "-n").returncode == 0 + assert _roar(tmp_path, "tracer", "use", "preload").returncode == 0 + return tmp_path + + +def test_a_shebang_script_records_the_script_not_its_interpreter(initialised: Path) -> None: + """The kernel rewrites cmdline to include the shebang interpreter, so /proc + reports `/usr/bin/env python3 ./train.sh` for a run of `./train.sh`.""" + script = initialised / "train.sh" + script.write_text("#!/usr/bin/env python3\nprint('hi')\n") + script.chmod(0o755) + + run = _roar(initialised, "run", "./train.sh") + + assert run.returncode == 0 + assert _latest_command(initialised) == ["./train.sh"] + + +def test_a_wrapper_command_is_recorded_as_given(initialised: Path) -> None: + run = _roar(initialised, "run", "env", "-u", "PYTHONPATH", sys.executable, "-c", "pass") + + assert run.returncode == 0 + assert _latest_command(initialised) == [ + "env", + "-u", + "PYTHONPATH", + sys.executable, + "-c", + "pass", + ] + + +def test_a_short_lived_command_still_records_its_argv(initialised: Path) -> None: + """A process that exits immediately may be a zombie by the time /proc is + read, which is where the recorded argv used to vary with machine load.""" + for _ in range(5): + run = _roar(initialised, "run", "/usr/bin/true") + + assert run.returncode == 0 + assert _latest_command(initialised) == ["/usr/bin/true"] From f82bcccc8303628945658e2e1554f9fb0f84586e Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Wed, 12 Aug 2026 19:53:15 +0000 Subject: [PATCH 41/52] ci: pin validated bpf-linker toolchain --- .github/workflows/ci.yml | 3 ++- .github/workflows/publish-pypi.yml | 3 ++- .github/workflows/publish-testpypi.yml | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2625d65d..02d145e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,8 @@ jobs: rustup component add rust-src --toolchain nightly - name: Install bpf-linker - run: cargo install bpf-linker + # Keep CI reproducible; upgrade alongside its required LLVM toolchain. + run: cargo install bpf-linker --version 0.10.4 --locked - name: Build and stage Rust binaries uses: ./.github/actions/build-rust-binaries diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 7ec96a34..0640ccc3 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -42,7 +42,8 @@ jobs: rustup component add rust-src --toolchain nightly - name: Install bpf-linker - run: cargo install bpf-linker + # Keep release builds aligned with the version validated in CI. + run: cargo install bpf-linker --version 0.10.4 --locked - name: Verify version matches release tag if: github.event_name == 'release' && matrix.arch == 'x86_64' diff --git a/.github/workflows/publish-testpypi.yml b/.github/workflows/publish-testpypi.yml index 7a81603f..9af997ec 100644 --- a/.github/workflows/publish-testpypi.yml +++ b/.github/workflows/publish-testpypi.yml @@ -35,7 +35,8 @@ jobs: rustup component add rust-src --toolchain nightly - name: Install bpf-linker - run: cargo install bpf-linker + # Keep release builds aligned with the version validated in CI. + run: cargo install bpf-linker --version 0.10.4 --locked - name: Build and stage Rust binaries uses: ./.github/actions/build-rust-binaries From 15cf7330e3429d42d6aa1e12a05eebcc34b01628 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Tue, 18 Aug 2026 19:35:28 +0000 Subject: [PATCH 42/52] test: keep the recorded-command tests off macOS protected launchers On Apple Silicon the system launchers are arm64e platform binaries, and dyld refuses to insert the arm64 preload dylib into them: incompatible architecture (have 'arm64', need 'arm64e') so the process aborts with 134 rather than running untraced. Both macOS jobs failed on the shebang case for that reason, via /usr/bin/env. Name the interpreter directly in the shebang instead. That keeps the case covered on macOS, and still exercises the kernel's cmdline rewrite on Linux -- verified it fails against a tracer built without the fix. The wrapper and short-lived cases need `env` and /usr/bin/true, which are protected with no substitute, so they carry the skip. Co-Authored-By: Claude Opus 5 (1M context) --- .../test_recorded_command_is_what_was_run.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_recorded_command_is_what_was_run.py b/tests/integration/test_recorded_command_is_what_was_run.py index 8a71dad5..b455bcdf 100644 --- a/tests/integration/test_recorded_command_is_what_was_run.py +++ b/tests/integration/test_recorded_command_is_what_was_run.py @@ -22,6 +22,14 @@ pytestmark = pytest.mark.integration +# On Apple Silicon the system launchers are arm64e platform binaries, and dyld +# refuses to insert the arm64 preload dylib into them: +# incompatible architecture (have 'arm64', need 'arm64e') +_MACOS_PROTECTED_BINARY = pytest.mark.skipif( + sys.platform == "darwin", + reason="macOS protected system binaries reject preload injection", +) + def _roar(cwd: Path, *args: str) -> subprocess.CompletedProcess[str]: return subprocess.run( @@ -51,9 +59,13 @@ def initialised(tmp_path: Path) -> Path: def test_a_shebang_script_records_the_script_not_its_interpreter(initialised: Path) -> None: """The kernel rewrites cmdline to include the shebang interpreter, so /proc - reports `/usr/bin/env python3 ./train.sh` for a run of `./train.sh`.""" + reports ` ./train.sh` for a run of `./train.sh`. + + The interpreter is named directly rather than via `/usr/bin/env` so this + keeps running on macOS, where the system launchers are protected. + """ script = initialised / "train.sh" - script.write_text("#!/usr/bin/env python3\nprint('hi')\n") + script.write_text(f"#!{sys.executable}\nprint('hi')\n") script.chmod(0o755) run = _roar(initialised, "run", "./train.sh") @@ -62,6 +74,7 @@ def test_a_shebang_script_records_the_script_not_its_interpreter(initialised: Pa assert _latest_command(initialised) == ["./train.sh"] +@_MACOS_PROTECTED_BINARY def test_a_wrapper_command_is_recorded_as_given(initialised: Path) -> None: run = _roar(initialised, "run", "env", "-u", "PYTHONPATH", sys.executable, "-c", "pass") @@ -76,6 +89,7 @@ def test_a_wrapper_command_is_recorded_as_given(initialised: Path) -> None: ] +@_MACOS_PROTECTED_BINARY def test_a_short_lived_command_still_records_its_argv(initialised: Path) -> None: """A process that exits immediately may be a zombie by the time /proc is read, which is where the recorded argv used to vary with machine load.""" From 83327edd63a05adf65214ba6abef40144ce51e4c Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Wed, 12 Aug 2026 19:54:24 +0000 Subject: [PATCH 43/52] fix: flush inject shards from fork workers --- roar/execution/runtime/inject/tracker.py | 32 +++++++++++++++++++ .../runtime/test_inject_log_merge.py | 30 +++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/roar/execution/runtime/inject/tracker.py b/roar/execution/runtime/inject/tracker.py index 832247c5..be096220 100644 --- a/roar/execution/runtime/inject/tracker.py +++ b/roar/execution/runtime/inject/tracker.py @@ -341,10 +341,42 @@ def __init__( def install(self) -> None: """Patch builtins and environ access for activity capture.""" + self._install_fork_worker_finalizer() builtins.open = self.tracking_open builtins.__import__ = self.tracking_import setattr(self._environ, _ENVIRON_GET_METHOD_NAME, self.patched_environ_get) + def _install_fork_worker_finalizer(self) -> None: + """Make multiprocessing fork workers emit their per-PID inject shard. + + ``multiprocessing`` workers terminate through ``os._exit``, bypassing + Python's ordinary atexit handlers. Its own shutdown path does run + child-local ``Finalize`` callbacks, so register one after each fork. The + existing parent-side shard merger then sees the worker report exactly as + intended by PR #265. + """ + try: + from multiprocessing import util as multiprocessing_util + + multiprocessing_util.register_after_fork( + self, RuntimeInjectionTracker._register_in_fork_child + ) + except Exception: + pass + + @staticmethod + def _register_in_fork_child(tracker: RuntimeInjectionTracker) -> None: + try: + from multiprocessing import util as multiprocessing_util + + multiprocessing_util.Finalize( + None, + tracker.write_log, + exitpriority=-100, + ) + except Exception: + pass + def tracking_open(self, *args, **kwargs): if is_suppressed(): return self._real_open(*args, **kwargs) diff --git a/tests/execution/runtime/test_inject_log_merge.py b/tests/execution/runtime/test_inject_log_merge.py index 021a0725..f9525349 100644 --- a/tests/execution/runtime/test_inject_log_merge.py +++ b/tests/execution/runtime/test_inject_log_merge.py @@ -8,6 +8,7 @@ from __future__ import annotations import json +import multiprocessing import os from roar.execution.runtime.inject.tracker import ( @@ -34,6 +35,10 @@ def _shard(base, pid, data): (base.parent / f"{base.name}.{pid}").write_text(json.dumps(data), encoding="utf-8") +def _record_fork_only_import(tracker): + tracker.imported_modules.add("fork_only_dependency") + + def test_write_log_writes_a_per_pid_shard_not_the_shared_file(tmp_path): log_path = tmp_path / "inject-log.json" _tracker(log_path).write_log() @@ -41,6 +46,31 @@ def test_write_log_writes_a_per_pid_shard_not_the_shared_file(tmp_path): assert (tmp_path / f"inject-log.json.{os.getpid()}").exists() # the shard is +def test_real_fork_worker_writes_its_own_pid_shard(tmp_path): + """Linux multiprocessing fork workers use os._exit, so ordinary atexit does + not run. The multiprocessing finalizer must write the worker's real shard.""" + if "fork" not in multiprocessing.get_all_start_methods(): + return + + log_path = tmp_path / "inject-log.json" + tracker = _tracker(log_path) + tracker._install_fork_worker_finalizer() + process = multiprocessing.get_context("fork").Process( + target=_record_fork_only_import, + args=(tracker,), + ) + process.start() + process.join(timeout=10) + + assert process.exitcode == 0 + worker_shard = tmp_path / f"inject-log.json.{process.pid}" + assert worker_shard.exists() + payload = json.loads(worker_shard.read_text()) + assert payload["pid"] == process.pid + assert "fork_only_dependency" in payload["imported_modules"] + assert not (tmp_path / f"inject-log.json.{os.getpid()}").exists() + + def test_worker_shard_does_not_clobber_the_workload_record(tmp_path): """MMA's litdata case: same command, a worker shard with argv ['-c'] and a subset of packages, plus the workload shard with the real command and the From 57b25df9f11187d448eef10fd93826c3b3b75395 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Wed, 12 Aug 2026 20:02:12 +0000 Subject: [PATCH 44/52] test: document forced-exit capture boundary --- .../runtime/test_inject_log_merge.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/execution/runtime/test_inject_log_merge.py b/tests/execution/runtime/test_inject_log_merge.py index f9525349..297b9ade 100644 --- a/tests/execution/runtime/test_inject_log_merge.py +++ b/tests/execution/runtime/test_inject_log_merge.py @@ -39,6 +39,10 @@ def _record_fork_only_import(tracker): tracker.imported_modules.add("fork_only_dependency") +def _exit_without_multiprocessing_cleanup(): + os._exit(0) + + def test_write_log_writes_a_per_pid_shard_not_the_shared_file(tmp_path): log_path = tmp_path / "inject-log.json" _tracker(log_path).write_log() @@ -71,6 +75,26 @@ def test_real_fork_worker_writes_its_own_pid_shard(tmp_path): assert not (tmp_path / f"inject-log.json.{os.getpid()}").exists() +def test_forced_os_exit_remains_outside_finalizer_guarantee(tmp_path): + """Document the lifecycle boundary: user code that calls os._exit bypasses + multiprocessing cleanup as well as atexit. Crash safety needs incremental + import journaling; the orderly-worker finalizer must not pretend otherwise.""" + if "fork" not in multiprocessing.get_all_start_methods(): + return + + log_path = tmp_path / "inject-log.json" + tracker = _tracker(log_path) + tracker._install_fork_worker_finalizer() + process = multiprocessing.get_context("fork").Process( + target=_exit_without_multiprocessing_cleanup + ) + process.start() + process.join(timeout=10) + + assert process.exitcode == 0 + assert not (tmp_path / f"inject-log.json.{process.pid}").exists() + + def test_worker_shard_does_not_clobber_the_workload_record(tmp_path): """MMA's litdata case: same command, a worker shard with argv ['-c'] and a subset of packages, plus the workload shard with the real command and the From 7fa2901344c5f50406ca676e36229c911a59fe89 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Tue, 18 Aug 2026 22:29:42 +0000 Subject: [PATCH 45/52] fix: also emit the shard when a fork worker is terminated Review found the finalizer covered only half the teardown paths. `Pool.__exit__` is `terminate()`, which SIGTERMs every worker, and `util._exit_function` does the same to surviving daemon children -- which is what DataLoader creates. SIGTERM's default disposition kills the process outright, so neither the Finalize callback nor atexit runs. Measured under real injection, 4 workers: with ctx.Pool(4) as p: p.map(...) 1 shard (0 of 4 workers) p.close(); p.join() 5 shards (4 of 4) So the common idiom -- and the `num_proc` case the finalizer's own docstring cites -- reported nothing at all. It now reports 4 of 4, and a daemon child is captured too. The handler is installed only in a fork child, and only where nothing else owns the signal, so a workload's own SIGTERM handling is never displaced. It restores the default disposition and re-raises, so the process still dies of SIGTERM and still reports exit status -15. One of our own handlers may be superseded, so a re-install or a second tracker does not leave a stale one writing the wrong shard. `signal` is imported in the parent so the child's import is a sys.modules hit; importing for the first time inside a fork child can deadlock on the import lock. Two tests, each verified to fail against the code without its fix: - the wiring test goes through `install()` rather than the private method. Deleting the single line that wires this into `install()` previously left all 125 tests passing -- and that line sits in the merge-conflict region with #287, so a conflict resolution could drop it silently. - the Pool test pins the terminated case. A third asserts a workload-owned SIGTERM handler is not displaced. The documented boundary now matches the code: termination by signal is covered; `os._exit` and SIGKILL are not, and cannot be without incremental import journaling. Co-Authored-By: Claude Opus 5 (1M context) --- roar/execution/runtime/inject/tracker.py | 55 +++++++++ .../runtime/test_inject_log_merge.py | 104 +++++++++++++++++- 2 files changed, 158 insertions(+), 1 deletion(-) diff --git a/roar/execution/runtime/inject/tracker.py b/roar/execution/runtime/inject/tracker.py index be096220..1358b4da 100644 --- a/roar/execution/runtime/inject/tracker.py +++ b/roar/execution/runtime/inject/tracker.py @@ -16,6 +16,9 @@ _ORIGINAL_ENVIRON_GET_ATTR = "_original_get" _ENVIRON_GET_METHOD_NAME = "get" +# Marks our own SIGTERM handler so a re-install can supersede it without +# displacing one the workload installed. +_SHARD_WRITER_ATTR = "_roar_shard_writer" # roar injects its own variables (ROAR_WRAP, ROAR_EXECUTION_BACKEND, # ROAR_RUNTIME_PYTHONPATH_ACTIVE, ...) into the traced process environment. @@ -354,8 +357,20 @@ def _install_fork_worker_finalizer(self) -> None: child-local ``Finalize`` callbacks, so register one after each fork. The existing parent-side shard merger then sees the worker report exactly as intended by PR #265. + + Workers are not always asked to stop, though -- see + ``_install_sigterm_shard_writer`` for the terminated case, which is what + ``with Pool(...)`` does. + + What remains uncovered, deliberately: a workload calling ``os._exit`` + directly, and SIGKILL. Neither can be intercepted; covering them needs + incremental import journaling rather than an exit hook. """ try: + # Imported here, in the parent, so the fork child's own `import + # signal` below is a sys.modules hit. Importing for the first time + # inside a fork child can deadlock on the import lock. + import signal # noqa: F401 from multiprocessing import util as multiprocessing_util multiprocessing_util.register_after_fork( @@ -376,6 +391,46 @@ def _register_in_fork_child(tracker: RuntimeInjectionTracker) -> None: ) except Exception: pass + tracker._install_sigterm_shard_writer() + + def _install_sigterm_shard_writer(self) -> None: + """Also emit the shard when a worker is *terminated* rather than joined. + + ``Pool.__exit__`` is ``terminate()``, which SIGTERMs every worker, and + ``util._exit_function`` does the same to surviving daemon children -- + which is what ``DataLoader`` creates. SIGTERM's default disposition kills + the process outright, so neither the finalizer above nor atexit runs. + Without this, ``with Pool(...) as p:`` -- the common idiom, and the + ``num_proc`` case in the docstring above -- reports nothing at all, while + ``close()``/``join()`` reports fine. + + Installed only in a fork child, and only when nothing else owns the + signal, so a workload's own SIGTERM handling is never displaced. The + default disposition is restored and re-raised so the process still dies + of SIGTERM and reports exit status -15 as the caller expects. + """ + try: + import signal + + current = signal.getsignal(signal.SIGTERM) + # Never displace a handler the workload owns. One of our own is + # fair game: a re-install, or a second tracker, should supersede it + # rather than leave the stale one writing the wrong shard. + if current is not signal.SIG_DFL and not getattr(current, _SHARD_WRITER_ATTR, False): + return + + def _write_shard_then_die(signum, frame): + with contextlib.suppress(Exception): + self.write_log() + signal.signal(signal.SIGTERM, signal.SIG_DFL) + os.kill(os.getpid(), signal.SIGTERM) + + setattr(_write_shard_then_die, _SHARD_WRITER_ATTR, True) + signal.signal(signal.SIGTERM, _write_shard_then_die) + except Exception: + # Not the main thread, no SIGTERM on this platform, etc. The + # finalizer path still covers orderly shutdown. + pass def tracking_open(self, *args, **kwargs): if is_suppressed(): diff --git a/tests/execution/runtime/test_inject_log_merge.py b/tests/execution/runtime/test_inject_log_merge.py index 297b9ade..a419f75a 100644 --- a/tests/execution/runtime/test_inject_log_merge.py +++ b/tests/execution/runtime/test_inject_log_merge.py @@ -7,9 +7,12 @@ from __future__ import annotations +import builtins import json import multiprocessing import os +import signal +import time from roar.execution.runtime.inject.tracker import ( RuntimeInjectionTracker, @@ -22,6 +25,10 @@ def handle_import(self, module_name, module): return None +class _FakeEnviron(dict): + """A dict that tolerates the attribute patching ``install()`` performs.""" + + def _tracker(log_path): return RuntimeInjectionTracker( {"ROAR_LOG_FILE": str(log_path)}, @@ -43,6 +50,97 @@ def _exit_without_multiprocessing_cleanup(): os._exit(0) +def _installable_tracker(log_path): + """A tracker whose environ tolerates ``install()``'s attribute patching.""" + return RuntimeInjectionTracker( + _FakeEnviron({"ROAR_LOG_FILE": str(log_path)}), + _FakeController(), + log_file=str(log_path), + inject_dir=str(log_path.parent / "inject"), + ) + + +def _noop_task(_): + return 1 + + +def _shards(tmp_path): + return sorted(p.name for p in tmp_path.glob("inject-log.json.*")) + + +def test_install_is_what_wires_the_fork_worker_finalizer(tmp_path): + """Go through ``install()``, not the private method. + + Calling ``_install_fork_worker_finalizer()`` directly passes even if the + single line wiring it into ``install()`` is deleted -- and that line sits in + the merge-conflict region with #287, so a conflict resolution could drop it + silently. This test is the only thing that would notice. + """ + if "fork" not in multiprocessing.get_all_start_methods(): + return + + log_path = tmp_path / "inject-log.json" + tracker = _installable_tracker(log_path) + + saved_open, saved_import = builtins.open, builtins.__import__ + try: + tracker.install() + process = multiprocessing.get_context("fork").Process( + target=_record_fork_only_import, + args=(tracker,), + ) + process.start() + process.join(timeout=10) + finally: + builtins.open, builtins.__import__ = saved_open, saved_import + + assert process.exitcode == 0 + assert (tmp_path / f"inject-log.json.{process.pid}").exists() + + +def test_pool_context_manager_workers_still_report(tmp_path): + """``with Pool(...)`` exits via ``terminate()``, which SIGTERMs the workers. + + SIGTERM's default disposition kills them outright, so neither the + multiprocessing finalizer nor atexit runs. This is the common idiom -- and + the ``num_proc`` case the finalizer's own docstring cites -- so it has to + report, not just the ``close()``/``join()`` shape. + """ + if "fork" not in multiprocessing.get_all_start_methods(): + return + + log_path = tmp_path / "inject-log.json" + tracker = _installable_tracker(log_path) + tracker._install_fork_worker_finalizer() + + context = multiprocessing.get_context("fork") + with context.Pool(2) as pool: + pool.map(_noop_task, range(2)) + + deadline = time.time() + 10 + while time.time() < deadline and len(_shards(tmp_path)) < 2: + time.sleep(0.05) + + assert len(_shards(tmp_path)) == 2, f"workers did not report: {_shards(tmp_path)}" + # The parent writes via atexit, which has not run yet. + assert f"inject-log.json.{os.getpid()}" not in _shards(tmp_path) + + +def test_a_workload_that_owns_sigterm_is_not_displaced(tmp_path): + """Capture must never take a signal the workload is already handling.""" + tracker = _installable_tracker(tmp_path / "inject-log.json") + + def _workload_handler(signum, frame): + return None + + previous = signal.signal(signal.SIGTERM, _workload_handler) + try: + tracker._install_sigterm_shard_writer() + assert signal.getsignal(signal.SIGTERM) is _workload_handler + finally: + signal.signal(signal.SIGTERM, previous) + + def test_write_log_writes_a_per_pid_shard_not_the_shared_file(tmp_path): log_path = tmp_path / "inject-log.json" _tracker(log_path).write_log() @@ -78,7 +176,11 @@ def test_real_fork_worker_writes_its_own_pid_shard(tmp_path): def test_forced_os_exit_remains_outside_finalizer_guarantee(tmp_path): """Document the lifecycle boundary: user code that calls os._exit bypasses multiprocessing cleanup as well as atexit. Crash safety needs incremental - import journaling; the orderly-worker finalizer must not pretend otherwise.""" + import journaling; the orderly-worker finalizer must not pretend otherwise. + + This is the honest remainder, not the whole gap. Termination by signal -- + which is what `with Pool(...)` does to its workers -- IS covered, by + `_install_sigterm_shard_writer`. SIGKILL is not, and cannot be.""" if "fork" not in multiprocessing.get_all_start_methods(): return From c6c9d14106210096aaa724a6782a359f1905aae5 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Wed, 19 Aug 2026 12:20:02 +0000 Subject: [PATCH 46/52] fix: capture killed fork workers without touching signal disposition Replaces the SIGTERM handler from the previous commit, which was worse than the bug it fixed. A Python signal handler only runs when the interpreter reaches a bytecode boundary. A worker inside a long C call -- BLAS, zlib, pickle -- latches the signal and never dies, and both `Pool._terminate_pool` and `util._exit_function` join workers with no timeout. So the handler could hang the workload indefinitely: worker in zlib.compress, on terminate() without roar exit -15, joined in 0.02s with the handler still alive after 8s, needed SIGKILL A missing shard is a thin record. A hung pipeline is a stopped campaign. Instead the fork child writes its shard eagerly, right after forking, and keeps the existing Finalize to rewrite the same per-PID path on an orderly exit. A worker that is killed -- SIGTERM, SIGKILL -- or that calls os._exit still contributes the state it inherited at fork, rather than nothing. An orderly worker upgrades that with whatever it imported while running. merge_inject_logs unions both, so the upgrade is free. This also covers strictly more than the handler did: SIGKILL and os._exit were previously uncoverable, and both now leave a valid shard. The boundary, stated honestly and pinned by a test: imports a worker makes *after* forking are lost if it is killed before exiting. Closing that needs incremental journaling, not an exit hook. Coverage for a worker killed while still bootstrapping is best-effort, so the pool test asserts against workers that actually ran a task rather than an exact shard count. Verified: hang gone; `with Pool(...)`, close()/join(), bare Process and daemon children all report; 127 runtime tests pass; removing the install() wiring, the eager write, or the Finalize each fails a test. Co-Authored-By: Claude Opus 5 (1M context) --- roar/execution/runtime/inject/tracker.py | 86 +++++++------------ .../runtime/test_inject_log_merge.py | 69 ++++++++------- 2 files changed, 63 insertions(+), 92 deletions(-) diff --git a/roar/execution/runtime/inject/tracker.py b/roar/execution/runtime/inject/tracker.py index 1358b4da..e2b70019 100644 --- a/roar/execution/runtime/inject/tracker.py +++ b/roar/execution/runtime/inject/tracker.py @@ -16,9 +16,6 @@ _ORIGINAL_ENVIRON_GET_ATTR = "_original_get" _ENVIRON_GET_METHOD_NAME = "get" -# Marks our own SIGTERM handler so a re-install can supersede it without -# displacing one the workload installed. -_SHARD_WRITER_ATTR = "_roar_shard_writer" # roar injects its own variables (ROAR_WRAP, ROAR_EXECUTION_BACKEND, # ROAR_RUNTIME_PYTHONPATH_ACTIVE, ...) into the traced process environment. @@ -358,19 +355,21 @@ def _install_fork_worker_finalizer(self) -> None: existing parent-side shard merger then sees the worker report exactly as intended by PR #265. - Workers are not always asked to stop, though -- see - ``_install_sigterm_shard_writer`` for the terminated case, which is what - ``with Pool(...)`` does. - - What remains uncovered, deliberately: a workload calling ``os._exit`` - directly, and SIGKILL. Neither can be intercepted; covering them needs - incremental import journaling rather than an exit hook. + Workers are not always asked to stop, though. ``Pool.__exit__`` is + ``terminate()``, which SIGTERMs every worker, and ``util._exit_function`` + does the same to surviving daemon children -- the ``DataLoader`` shape. + A killed worker runs no exit hook at all, so the child also writes its + shard *immediately* after forking; see ``_register_in_fork_child``. + + Deliberately NOT done here: installing a SIGTERM handler. A Python + signal handler only runs when the interpreter reaches a bytecode + boundary, so a worker inside a long C call (BLAS, zlib, pickle) would + latch the signal and never die -- and both ``Pool._terminate_pool`` and + ``util._exit_function`` join workers with no timeout. Measured: a worker + in ``zlib.compress`` went from exit -15 in 0.02s to still alive after + 8s. Hanging the workload is far worse than a thin shard. """ try: - # Imported here, in the parent, so the fork child's own `import - # signal` below is a sys.modules hit. Importing for the first time - # inside a fork child can deadlock on the import lock. - import signal # noqa: F401 from multiprocessing import util as multiprocessing_util multiprocessing_util.register_after_fork( @@ -381,7 +380,22 @@ def _install_fork_worker_finalizer(self) -> None: @staticmethod def _register_in_fork_child(tracker: RuntimeInjectionTracker) -> None: - try: + """Give the child a shard now, and a complete one if it exits orderly. + + The eager write is what survives a worker that is killed rather than + joined: it holds the state inherited at fork, which is the parent's + whole import set. The finalizer then rewrites the same per-PID path on + an orderly exit, upgrading it with whatever the worker imported while it + ran. Both paths are union-merged by ``merge_inject_logs``, so the + upgrade is free and a killed worker still contributes. + + The remaining boundary, stated plainly: imports a worker makes *after* + forking are lost if it is killed before exiting. Closing that needs + incremental journaling, not an exit hook. + """ + with contextlib.suppress(Exception): + tracker.write_log() + with contextlib.suppress(Exception): from multiprocessing import util as multiprocessing_util multiprocessing_util.Finalize( @@ -389,48 +403,6 @@ def _register_in_fork_child(tracker: RuntimeInjectionTracker) -> None: tracker.write_log, exitpriority=-100, ) - except Exception: - pass - tracker._install_sigterm_shard_writer() - - def _install_sigterm_shard_writer(self) -> None: - """Also emit the shard when a worker is *terminated* rather than joined. - - ``Pool.__exit__`` is ``terminate()``, which SIGTERMs every worker, and - ``util._exit_function`` does the same to surviving daemon children -- - which is what ``DataLoader`` creates. SIGTERM's default disposition kills - the process outright, so neither the finalizer above nor atexit runs. - Without this, ``with Pool(...) as p:`` -- the common idiom, and the - ``num_proc`` case in the docstring above -- reports nothing at all, while - ``close()``/``join()`` reports fine. - - Installed only in a fork child, and only when nothing else owns the - signal, so a workload's own SIGTERM handling is never displaced. The - default disposition is restored and re-raised so the process still dies - of SIGTERM and reports exit status -15 as the caller expects. - """ - try: - import signal - - current = signal.getsignal(signal.SIGTERM) - # Never displace a handler the workload owns. One of our own is - # fair game: a re-install, or a second tracker, should supersede it - # rather than leave the stale one writing the wrong shard. - if current is not signal.SIG_DFL and not getattr(current, _SHARD_WRITER_ATTR, False): - return - - def _write_shard_then_die(signum, frame): - with contextlib.suppress(Exception): - self.write_log() - signal.signal(signal.SIGTERM, signal.SIG_DFL) - os.kill(os.getpid(), signal.SIGTERM) - - setattr(_write_shard_then_die, _SHARD_WRITER_ATTR, True) - signal.signal(signal.SIGTERM, _write_shard_then_die) - except Exception: - # Not the main thread, no SIGTERM on this platform, etc. The - # finalizer path still covers orderly shutdown. - pass def tracking_open(self, *args, **kwargs): if is_suppressed(): diff --git a/tests/execution/runtime/test_inject_log_merge.py b/tests/execution/runtime/test_inject_log_merge.py index a419f75a..6c73f844 100644 --- a/tests/execution/runtime/test_inject_log_merge.py +++ b/tests/execution/runtime/test_inject_log_merge.py @@ -11,8 +11,6 @@ import json import multiprocessing import os -import signal -import time from roar.execution.runtime.inject.tracker import ( RuntimeInjectionTracker, @@ -46,7 +44,10 @@ def _record_fork_only_import(tracker): tracker.imported_modules.add("fork_only_dependency") -def _exit_without_multiprocessing_cleanup(): +def _record_then_force_exit(tracker): + # Recorded after fork, so only an exit hook could capture it -- and + # os._exit runs none. + tracker.imported_modules.add("fork_only_dependency") os._exit(0) @@ -60,8 +61,8 @@ def _installable_tracker(log_path): ) -def _noop_task(_): - return 1 +def _report_pid(_): + return os.getpid() def _shards(tmp_path): @@ -105,6 +106,11 @@ def test_pool_context_manager_workers_still_report(tmp_path): multiprocessing finalizer nor atexit runs. This is the common idiom -- and the ``num_proc`` case the finalizer's own docstring cites -- so it has to report, not just the ``close()``/``join()`` shape. + + Asserted against the workers that actually ran a task, since those + demonstrably got through the after-fork hook. A worker forked and killed + while still bootstrapping may write nothing: the eager write makes this + best-effort, not a guarantee, and an exact shard count would be flaky. """ if "fork" not in multiprocessing.get_all_start_methods(): return @@ -115,32 +121,17 @@ def test_pool_context_manager_workers_still_report(tmp_path): context = multiprocessing.get_context("fork") with context.Pool(2) as pool: - pool.map(_noop_task, range(2)) + worker_pids = set(pool.map(_report_pid, range(8))) - deadline = time.time() + 10 - while time.time() < deadline and len(_shards(tmp_path)) < 2: - time.sleep(0.05) - - assert len(_shards(tmp_path)) == 2, f"workers did not report: {_shards(tmp_path)}" + assert worker_pids, "no worker ran a task" + for pid in worker_pids: + assert (tmp_path / f"inject-log.json.{pid}").exists(), ( + f"worker {pid} ran a task but never reported; shards: {_shards(tmp_path)}" + ) # The parent writes via atexit, which has not run yet. assert f"inject-log.json.{os.getpid()}" not in _shards(tmp_path) -def test_a_workload_that_owns_sigterm_is_not_displaced(tmp_path): - """Capture must never take a signal the workload is already handling.""" - tracker = _installable_tracker(tmp_path / "inject-log.json") - - def _workload_handler(signum, frame): - return None - - previous = signal.signal(signal.SIGTERM, _workload_handler) - try: - tracker._install_sigterm_shard_writer() - assert signal.getsignal(signal.SIGTERM) is _workload_handler - finally: - signal.signal(signal.SIGTERM, previous) - - def test_write_log_writes_a_per_pid_shard_not_the_shared_file(tmp_path): log_path = tmp_path / "inject-log.json" _tracker(log_path).write_log() @@ -173,14 +164,18 @@ def test_real_fork_worker_writes_its_own_pid_shard(tmp_path): assert not (tmp_path / f"inject-log.json.{os.getpid()}").exists() -def test_forced_os_exit_remains_outside_finalizer_guarantee(tmp_path): - """Document the lifecycle boundary: user code that calls os._exit bypasses - multiprocessing cleanup as well as atexit. Crash safety needs incremental - import journaling; the orderly-worker finalizer must not pretend otherwise. +def test_a_forced_exit_keeps_the_fork_time_shard_but_loses_later_imports(tmp_path): + """Document the lifecycle boundary precisely. + + A worker that calls ``os._exit`` bypasses multiprocessing cleanup as well as + atexit, so no exit hook runs for it -- and the same is true of one killed by + SIGTERM or SIGKILL. The eager write at fork means such a worker still + contributes the state it inherited, rather than nothing at all. - This is the honest remainder, not the whole gap. Termination by signal -- - which is what `with Pool(...)` does to its workers -- IS covered, by - `_install_sigterm_shard_writer`. SIGKILL is not, and cannot be.""" + What is lost is what it imported *after* forking. That is the honest + remainder, and closing it needs incremental import journaling rather than an + exit hook. This test pins both halves so neither claim drifts. + """ if "fork" not in multiprocessing.get_all_start_methods(): return @@ -188,13 +183,17 @@ def test_forced_os_exit_remains_outside_finalizer_guarantee(tmp_path): tracker = _tracker(log_path) tracker._install_fork_worker_finalizer() process = multiprocessing.get_context("fork").Process( - target=_exit_without_multiprocessing_cleanup + target=_record_then_force_exit, + args=(tracker,), ) process.start() process.join(timeout=10) assert process.exitcode == 0 - assert not (tmp_path / f"inject-log.json.{process.pid}").exists() + worker_shard = tmp_path / f"inject-log.json.{process.pid}" + assert worker_shard.exists(), "the fork-time snapshot should survive a forced exit" + payload = json.loads(worker_shard.read_text()) + assert "fork_only_dependency" not in payload["imported_modules"] def test_worker_shard_does_not_clobber_the_workload_record(tmp_path): From fb00383db0b668d2978739fecc28689198ad65d3 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Wed, 12 Aug 2026 18:44:32 +0000 Subject: [PATCH 47/52] fix: fail closed when Python package capture is suppressed --- roar/application/reproducibility/report.py | 7 +- roar/core/models/provenance.py | 3 + roar/execution/provenance/assembler.py | 1 + roar/execution/provenance/data_loader.py | 3 + roar/execution/provenance/service.py | 24 ++++++ roar/execution/recording/job_recording.py | 2 + .../reproduction/pipeline_metadata.py | 11 +++ roar/execution/runtime/coordinator.py | 7 ++ .../reproducibility/test_report.py | 23 ++++++ .../test_python_capture_fail_closed.py | 77 +++++++++++++++++++ tests/unit/test_tracer_data_loader.py | 15 ++++ 11 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_python_capture_fail_closed.py diff --git a/roar/application/reproducibility/report.py b/roar/application/reproducibility/report.py index 0665a1d8..2ddb655d 100644 --- a/roar/application/reproducibility/report.py +++ b/roar/application/reproducibility/report.py @@ -180,8 +180,11 @@ def runtime_captured(pipeline) -> bool: from ...execution.reproduction.pipeline_metadata import PipelineMetadataParser try: - runtime = PipelineMetadataParser().first_runtime(pipeline.build_steps, pipeline.run_steps) - return bool((runtime.get("python") or {}).get("version")) + parser = PipelineMetadataParser() + runtime = parser.first_runtime(pipeline.build_steps, pipeline.run_steps) + return bool( + (runtime.get("python") or {}).get("version") + ) and parser.python_capture_complete(pipeline.build_steps, pipeline.run_steps) except Exception: return False diff --git a/roar/core/models/provenance.py b/roar/core/models/provenance.py index cd992605..57c1e5ee 100644 --- a/roar/core/models/provenance.py +++ b/roar/core/models/provenance.py @@ -67,6 +67,9 @@ class PythonInjectData(RoarBaseModel): installed_packages: dict[str, str] = Field(default_factory=dict) python_version: str = "" python_implementation: str = "" + # Health of the sitecustomize package-capture channel. A missing/invalid + # Python capture must not be confused with a successful empty package set. + capture_status: str = "missing" @computed_field # type: ignore[prop-decorator] @property diff --git a/roar/execution/provenance/assembler.py b/roar/execution/provenance/assembler.py index 7600a44d..17c43829 100644 --- a/roar/execution/provenance/assembler.py +++ b/roar/execution/provenance/assembler.py @@ -103,6 +103,7 @@ def assemble(self, ctx: ProvenanceContext, config: dict[str, Any]) -> dict[str, }, "processes": ctx.process_summary, "runtime": self._runtime_to_dict(ctx.runtime_info), + "python_capture": ctx.python_data.capture_status, } # Add analyzer results diff --git a/roar/execution/provenance/data_loader.py b/roar/execution/provenance/data_loader.py index e57d4b02..b559e6a5 100644 --- a/roar/execution/provenance/data_loader.py +++ b/roar/execution/provenance/data_loader.py @@ -195,6 +195,7 @@ def load_python_data(self, path: str | None) -> PythonInjectData: return PythonInjectData( sys_prefix=sys.prefix, sys_base_prefix=sys.base_prefix, + capture_status="missing", ) try: @@ -206,6 +207,7 @@ def load_python_data(self, path: str | None) -> PythonInjectData: return PythonInjectData( sys_prefix=sys.prefix, sys_base_prefix=sys.base_prefix, + capture_status="invalid", ) return PythonInjectData( @@ -219,4 +221,5 @@ def load_python_data(self, path: str | None) -> PythonInjectData: installed_packages=data.get("installed_packages", {}), python_version=data.get("python_version", ""), python_implementation=data.get("python_implementation", ""), + capture_status="complete", ) diff --git a/roar/execution/provenance/service.py b/roar/execution/provenance/service.py index 80c1a391..9b8c1ea0 100644 --- a/roar/execution/provenance/service.py +++ b/roar/execution/provenance/service.py @@ -5,6 +5,7 @@ """ import os +import re import shutil from datetime import datetime, timezone from typing import Any @@ -125,6 +126,10 @@ def collect( len(tracer_data.processes), ) python_data = self._data_loader.load_python_data(python_log_path) + if python_data.capture_status == "missing" and not self._contains_python_process( + tracer_data.processes + ): + python_data.capture_status = "not-applicable" self.logger.debug( "Python data loaded: modules=%d, packages=%d", len(python_data.modules_files), @@ -260,6 +265,7 @@ def collect( "shared_libs": python_data.shared_libs, "used_packages": python_data.used_packages, "installed_packages": python_data.installed_packages, + "capture_status": python_data.capture_status, }, } analyzer_results = analyzers.run_analyzers(analyzer_context, config=config) @@ -286,6 +292,24 @@ def collect( self.logger.debug("Provenance collection complete") return result + @staticmethod + def _contains_python_process(processes: list[dict[str, Any]]) -> bool: + """Whether the native trace observed a Python interpreter process.""" + for process in processes: + command = process.get("command") or [] + if isinstance(command, str): + command = [command] + if not isinstance(command, list) or not command: + continue + # Wrapper processes such as `env PYTHONPATH=. python ...` may be + # the only process entry emitted by preload, so inspect every argv + # token for an interpreter executable rather than argv[0] alone. + for token in command: + executable = os.path.basename(str(token)).lower() + if re.fullmatch(r"python(?:\d+(?:\.\d+)*)?(?:\.exe)?", executable): + return True + return False + def _resolve_exec_program(self, command: list[str] | None) -> str | None: """Resolve the run's exec'd program (the user's argv[0]) to an abspath. diff --git a/roar/execution/recording/job_recording.py b/roar/execution/recording/job_recording.py index 914ac6c2..4d97de8b 100644 --- a/roar/execution/recording/job_recording.py +++ b/roar/execution/recording/job_recording.py @@ -392,6 +392,8 @@ def _build_metadata_json( metadata["packages"] = prov["executables"]["packages"] if prov.get("runtime"): metadata["runtime"] = prov["runtime"] + if prov.get("python_capture"): + metadata["python_capture"] = prov["python_capture"] if prov.get("analysis"): metadata["analysis"] = prov["analysis"] metadata["git"] = git_info diff --git a/roar/execution/reproduction/pipeline_metadata.py b/roar/execution/reproduction/pipeline_metadata.py index 91f256d2..117783b8 100644 --- a/roar/execution/reproduction/pipeline_metadata.py +++ b/roar/execution/reproduction/pipeline_metadata.py @@ -68,6 +68,17 @@ def first_runtime(self, build_steps: list[dict], run_steps: list[dict]) -> dict[ return runtime return {} + def python_capture_complete(self, build_steps: list[dict], run_steps: list[dict]) -> bool: + """False when new lineage explicitly reports failed Python capture. + + Older lineage has no marker and remains backward-compatible. + """ + for step in [*build_steps, *run_steps]: + metadata = self._normalize_metadata(step.get("metadata")) + if metadata.get("python_capture") in {"missing", "invalid"}: + return False + return True + def summarize_requirements( self, build_steps: list[dict], run_steps: list[dict] ) -> RequirementSummary: diff --git a/roar/execution/runtime/coordinator.py b/roar/execution/runtime/coordinator.py index 8a589c59..70ca873e 100644 --- a/roar/execution/runtime/coordinator.py +++ b/roar/execution/runtime/coordinator.py @@ -291,6 +291,13 @@ def stop_runtime_resources(exit_code: int | None) -> RuntimeObservationBundle: collect_dropped_paths=(ctx.verbosity == "debug"), command=list(ctx.command), ) + if prov.get("python_capture") in {"missing", "invalid"}: + self.presenter.print_error( + "warning: Python package capture did not complete; this job's package list is " + "incomplete and its runtime reproducibility check will fail.\n" + " Avoid replacing/removing PYTHONPATH, `env -i`, and Python -E/-I/-S. " + "Use `env -C python ...` when only a working-directory change is needed." + ) t_prov_end = time.perf_counter() n_read = len(prov.get("data", {}).get("read_files", [])) n_written = len(prov.get("data", {}).get("written_files", [])) diff --git a/tests/application/reproducibility/test_report.py b/tests/application/reproducibility/test_report.py index cbf7b853..ffe4b8c0 100644 --- a/tests/application/reproducibility/test_report.py +++ b/tests/application/reproducibility/test_report.py @@ -9,10 +9,33 @@ build_report, is_shareable_remote, render_report, + runtime_captured, untracked_artifact_dirs, ) +class _Pipeline: + def __init__(self, metadata): + self.build_steps = [] + self.run_steps = [{"metadata": metadata}] + + +def test_runtime_capture_rejects_explicitly_missing_python_injection(): + pipeline = _Pipeline( + {"runtime": {"python": {"version": "3.12.1"}}, "python_capture": "missing"} + ) + assert runtime_captured(pipeline) is False + + +def test_runtime_capture_accepts_complete_and_legacy_lineage(): + complete = _Pipeline( + {"runtime": {"python": {"version": "3.12.1"}}, "python_capture": "complete"} + ) + legacy = _Pipeline({"runtime": {"python": {"version": "3.12.1"}}}) + assert runtime_captured(complete) is True + assert runtime_captured(legacy) is True + + def _full_report(**overrides): # A register/put-style report: every fact supplied, so all checks render # (including the receipt-only `paths_tracked` and `on_glaas`). diff --git a/tests/integration/test_python_capture_fail_closed.py b/tests/integration/test_python_capture_fail_closed.py new file mode 100644 index 00000000..f73c8a6a --- /dev/null +++ b/tests/integration/test_python_capture_fail_closed.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import json +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.integration + + +def _roar(cwd: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-m", "roar", *args], + cwd=cwd, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + +def _latest_metadata(cwd: Path) -> dict: + connection = sqlite3.connect(cwd / ".roar" / "roar.db") + row = connection.execute("SELECT metadata FROM jobs ORDER BY id DESC LIMIT 1").fetchone() + connection.close() + assert row is not None + return json.loads(row[0]) + + +@pytest.mark.parametrize( + "command", + [ + ["env", "PYTHONPATH=.", sys.executable, "-c", "import click"], + ["env", "-u", "PYTHONPATH", sys.executable, "-c", "import click"], + [sys.executable, "-E", "-c", "import click"], + [sys.executable, "-I", "-c", "import click"], + [sys.executable, "-S", "-c", "pass"], + ["sh", "-c", f"PYTHONPATH=. {sys.executable} -c 'import click'"], + ], + ids=["env-replace", "env-unset", "python-E", "python-I", "python-S", "shell-replace"], +) +def test_suppressed_injection_warns_and_records_failed_capture( + tmp_path: Path, command: list[str] +) -> None: + assert _roar(tmp_path, "init", "-n").returncode == 0 + assert _roar(tmp_path, "tracer", "use", "preload").returncode == 0 + + run = _roar(tmp_path, "run", *command) + + assert run.returncode == 0 + assert "Python package capture did not complete" in run.stderr + assert _latest_metadata(tmp_path)["python_capture"] == "missing" + + +def test_successful_python_capture_has_no_warning(tmp_path: Path) -> None: + assert _roar(tmp_path, "init", "-n").returncode == 0 + assert _roar(tmp_path, "tracer", "use", "preload").returncode == 0 + + run = _roar(tmp_path, "run", sys.executable, "-c", "import click") + + assert run.returncode == 0 + assert "Python package capture did not complete" not in run.stderr + assert _latest_metadata(tmp_path)["python_capture"] == "complete" + + +def test_non_python_command_is_not_misreported(tmp_path: Path) -> None: + assert _roar(tmp_path, "init", "-n").returncode == 0 + assert _roar(tmp_path, "tracer", "use", "preload").returncode == 0 + + run = _roar(tmp_path, "run", "/bin/true") + + assert run.returncode == 0 + assert "Python package capture did not complete" not in run.stderr + assert _latest_metadata(tmp_path)["python_capture"] == "not-applicable" diff --git a/tests/unit/test_tracer_data_loader.py b/tests/unit/test_tracer_data_loader.py index b593a6be..ff37d3b6 100644 --- a/tests/unit/test_tracer_data_loader.py +++ b/tests/unit/test_tracer_data_loader.py @@ -154,6 +154,21 @@ def test_preserves_thread_aware_file_contract_fields(self, tmp_path: Path) -> No class TestLoadPythonData: + def test_missing_log_is_distinct_from_complete_empty_capture(self, tmp_path: Path) -> None: + missing = DataLoaderService().load_python_data(None) + assert missing.capture_status == "missing" + + log_path = tmp_path / "inject-log.json" + _write_json(log_path, {}) + complete = DataLoaderService().load_python_data(str(log_path)) + assert complete.capture_status == "complete" + + def test_invalid_log_is_reported(self, tmp_path: Path) -> None: + log_path = tmp_path / "inject-log.json" + log_path.write_text("{not-json", encoding="utf-8") + + assert DataLoaderService().load_python_data(str(log_path)).capture_status == "invalid" + def test_python_identity_keys_flow_through(self, tmp_path: Path) -> None: """python_version / python_implementation make it from JSON into the model.""" log_path = tmp_path / "inject-log.json" From 90b94378cd128a13d943bdf0bec57a75d19e740a Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Wed, 12 Aug 2026 20:03:23 +0000 Subject: [PATCH 48/52] test: skip protected macOS preload launchers --- .../test_python_capture_fail_closed.py | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_python_capture_fail_closed.py b/tests/integration/test_python_capture_fail_closed.py index f73c8a6a..598dc5d6 100644 --- a/tests/integration/test_python_capture_fail_closed.py +++ b/tests/integration/test_python_capture_fail_closed.py @@ -10,6 +10,11 @@ pytestmark = pytest.mark.integration +_MACOS_PROTECTED_BINARY = pytest.mark.skipif( + sys.platform == "darwin", + reason="macOS protected system binaries reject preload before the workload starts", +) + def _roar(cwd: Path, *args: str) -> subprocess.CompletedProcess[str]: return subprocess.run( @@ -33,14 +38,25 @@ def _latest_metadata(cwd: Path) -> dict: @pytest.mark.parametrize( "command", [ - ["env", "PYTHONPATH=.", sys.executable, "-c", "import click"], - ["env", "-u", "PYTHONPATH", sys.executable, "-c", "import click"], - [sys.executable, "-E", "-c", "import click"], - [sys.executable, "-I", "-c", "import click"], - [sys.executable, "-S", "-c", "pass"], - ["sh", "-c", f"PYTHONPATH=. {sys.executable} -c 'import click'"], + pytest.param( + ["env", "PYTHONPATH=.", sys.executable, "-c", "import click"], + marks=_MACOS_PROTECTED_BINARY, + id="env-replace", + ), + pytest.param( + ["env", "-u", "PYTHONPATH", sys.executable, "-c", "import click"], + marks=_MACOS_PROTECTED_BINARY, + id="env-unset", + ), + pytest.param([sys.executable, "-E", "-c", "import click"], id="python-E"), + pytest.param([sys.executable, "-I", "-c", "import click"], id="python-I"), + pytest.param([sys.executable, "-S", "-c", "pass"], id="python-S"), + pytest.param( + ["sh", "-c", f"PYTHONPATH=. {sys.executable} -c 'import click'"], + marks=_MACOS_PROTECTED_BINARY, + id="shell-replace", + ), ], - ids=["env-replace", "env-unset", "python-E", "python-I", "python-S", "shell-replace"], ) def test_suppressed_injection_warns_and_records_failed_capture( tmp_path: Path, command: list[str] From 97c7d6c17507327d80ad610ae06c9b746884aeb6 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Tue, 18 Aug 2026 17:03:23 +0000 Subject: [PATCH 49/52] test: resolve `true` instead of hardcoding /bin/true /bin/true exists on most Linux distributions but not on macOS, where the binary is only at /usr/bin/true. `roar run /bin/true` therefore exited 127 on both macOS runners and read as a roar failure rather than a missing test fixture. Resolve it via PATH so the test exercises what it means to -- a non-Python command -- on either platform. Co-Authored-By: Claude Opus 5 (1M context) --- tests/integration/test_python_capture_fail_closed.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_python_capture_fail_closed.py b/tests/integration/test_python_capture_fail_closed.py index 598dc5d6..1c45e0a0 100644 --- a/tests/integration/test_python_capture_fail_closed.py +++ b/tests/integration/test_python_capture_fail_closed.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import shutil import sqlite3 import subprocess import sys @@ -15,6 +16,10 @@ reason="macOS protected system binaries reject preload before the workload starts", ) +# `true` is /bin/true on most Linux distributions but only /usr/bin/true on +# macOS, where a hardcoded /bin/true exits 127 and looks like a roar failure. +_TRUE_BINARY = shutil.which("true") or "/usr/bin/true" + def _roar(cwd: Path, *args: str) -> subprocess.CompletedProcess[str]: return subprocess.run( @@ -86,7 +91,7 @@ def test_non_python_command_is_not_misreported(tmp_path: Path) -> None: assert _roar(tmp_path, "init", "-n").returncode == 0 assert _roar(tmp_path, "tracer", "use", "preload").returncode == 0 - run = _roar(tmp_path, "run", "/bin/true") + run = _roar(tmp_path, "run", _TRUE_BINARY) assert run.returncode == 0 assert "Python package capture did not complete" not in run.stderr From 818eb07906d087977b0a31c04ccc1e09119755fc Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Tue, 18 Aug 2026 17:17:54 +0000 Subject: [PATCH 50/52] test: skip the non-Python capture case on macOS /usr/bin/true is SIP-protected, so roar's preflight correctly refuses to preload-inject into it and exits 1: Tracer preflight failed for 'preload': macOS protected binary blocks preload injection That is the condition _MACOS_PROTECTED_BINARY already documents for the other launchers in this file, so mark this case the same way. The previous /bin/true exited 127 before ever reaching preflight, which masked it. Co-Authored-By: Claude Opus 5 (1M context) --- tests/integration/test_python_capture_fail_closed.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_python_capture_fail_closed.py b/tests/integration/test_python_capture_fail_closed.py index 1c45e0a0..e555eee3 100644 --- a/tests/integration/test_python_capture_fail_closed.py +++ b/tests/integration/test_python_capture_fail_closed.py @@ -18,6 +18,7 @@ # `true` is /bin/true on most Linux distributions but only /usr/bin/true on # macOS, where a hardcoded /bin/true exits 127 and looks like a roar failure. +# On macOS it is SIP-protected either way, so its test carries the skip above. _TRUE_BINARY = shutil.which("true") or "/usr/bin/true" @@ -87,6 +88,7 @@ def test_successful_python_capture_has_no_warning(tmp_path: Path) -> None: assert _latest_metadata(tmp_path)["python_capture"] == "complete" +@_MACOS_PROTECTED_BINARY def test_non_python_command_is_not_misreported(tmp_path: Path) -> None: assert _roar(tmp_path, "init", "-n").returncode == 0 assert _roar(tmp_path, "tracer", "use", "preload").returncode == 0 From f20ecb0813051e2aa7f567532e683327674a4a27 Mon Sep 17 00:00:00 2001 From: Trevor Basinger Date: Tue, 18 Aug 2026 19:01:47 +0000 Subject: [PATCH 51/52] fix(release): preserve the Linux wheel baseline --- .github/workflows/ci.yml | 15 +++++++++++++-- .github/workflows/publish-pypi.yml | 16 ++++++++++++++-- .github/workflows/publish-testpypi.yml | 16 ++++++++++++++-- scripts/build_wheel_with_bins.sh | 14 ++++++++++++++ scripts/ci/verify_wheel_contents.py | 15 +++++++++------ 5 files changed, 64 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 235e801c..ea511b18 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -360,7 +360,7 @@ jobs: - name: Install build tools run: | python -m pip install --upgrade pip - pip install maturin + pip install 'maturin[zig]' - name: Download Rust binaries uses: actions/download-artifact@v4 @@ -378,7 +378,18 @@ jobs: - name: Build abi3 wheel env: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-deployment-target }} - run: maturin build --release --manifest-path rust/crates/artifact-hash-py/Cargo.toml --out dist + ROAR_WHEEL_PLATFORM: ${{ matrix.platform }} + run: | + args=( + build + --release + --manifest-path rust/crates/artifact-hash-py/Cargo.toml + --out dist + ) + if [[ "$ROAR_WHEEL_PLATFORM" == "linux" ]]; then + args+=(--zig --compatibility manylinux_2_17) + fi + maturin "${args[@]}" - name: Install uv run: | diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 7ec96a34..5a83b395 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -164,7 +164,7 @@ jobs: - name: Install build tools run: | python -m pip install --upgrade pip - pip install maturin + pip install 'maturin[zig]' - name: Download Rust binaries artifact uses: actions/download-artifact@v4 @@ -184,7 +184,19 @@ jobs: # macOS floor on macOS jobs so the single abi3 build host doesn't drift it. env: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-deployment-target }} - run: maturin build --release --manifest-path rust/crates/artifact-hash-py/Cargo.toml --interpreter python --out dist + ROAR_WHEEL_PLATFORM: ${{ matrix.platform }} + run: | + args=( + build + --release + --manifest-path rust/crates/artifact-hash-py/Cargo.toml + --interpreter python + --out dist + ) + if [[ "$ROAR_WHEEL_PLATFORM" == "linux" ]]; then + args+=(--zig --compatibility manylinux_2_17) + fi + maturin "${args[@]}" - name: Verify wheel contains native extension and binaries env: diff --git a/.github/workflows/publish-testpypi.yml b/.github/workflows/publish-testpypi.yml index 7a81603f..7ff66d5b 100644 --- a/.github/workflows/publish-testpypi.yml +++ b/.github/workflows/publish-testpypi.yml @@ -131,7 +131,7 @@ jobs: - name: Install build tools run: | python -m pip install --upgrade pip - pip install maturin + pip install 'maturin[zig]' - name: Download Rust binaries artifact uses: actions/download-artifact@v4 @@ -151,7 +151,19 @@ jobs: # macOS floor on macOS jobs so the single abi3 build host doesn't drift it. env: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-deployment-target }} - run: maturin build --release --manifest-path rust/crates/artifact-hash-py/Cargo.toml --interpreter python --out dist + ROAR_WHEEL_PLATFORM: ${{ matrix.platform }} + run: | + args=( + build + --release + --manifest-path rust/crates/artifact-hash-py/Cargo.toml + --interpreter python + --out dist + ) + if [[ "$ROAR_WHEEL_PLATFORM" == "linux" ]]; then + args+=(--zig --compatibility manylinux_2_17) + fi + maturin "${args[@]}" - name: Verify wheel contains native extension and binaries env: diff --git a/scripts/build_wheel_with_bins.sh b/scripts/build_wheel_with_bins.sh index 21d36984..8658b984 100755 --- a/scripts/build_wheel_with_bins.sh +++ b/scripts/build_wheel_with_bins.sh @@ -139,6 +139,20 @@ resolve_built_artifact() { } build_python_wheel() { + if [[ "$(uname -s)" == "Linux" ]]; then + echo "▶ Building portable manylinux_2_17 wheel with maturin and Zig..." + ( + cd "$ROOT_DIR" + uvx --from 'maturin[zig]' maturin build \ + --release \ + --zig \ + --compatibility manylinux_2_17 \ + --manifest-path rust/crates/artifact-hash-py/Cargo.toml \ + --out "$OUT_DIR" + ) + return + fi + if command -v uv >/dev/null 2>&1; then echo "▶ Building wheel with uv..." uv build --wheel --out-dir "$OUT_DIR" diff --git a/scripts/ci/verify_wheel_contents.py b/scripts/ci/verify_wheel_contents.py index 2a37073d..1d189f01 100644 --- a/scripts/ci/verify_wheel_contents.py +++ b/scripts/ci/verify_wheel_contents.py @@ -57,11 +57,13 @@ def main() -> None: if missing_bins: raise SystemExit(f"Missing binaries in wheel: {missing_bins}") - has_native = any( - name.startswith("roar/_hash_native") - and (name.endswith(".so") or name.endswith(".pyd") or name.endswith(".dylib")) + native_extensions = { + name for name in names - ) + if name.startswith("roar/_hash_native") + and (name.endswith(".so") or name.endswith(".pyd") or name.endswith(".dylib")) + } + has_native = bool(native_extensions) if not has_native: raise SystemExit("Missing native hash extension in wheel (roar/_hash_native*)") @@ -74,9 +76,10 @@ def main() -> None: raise SystemExit("Missing preload interposer library in wheel (roar/bin/libroar*_preload*)") if platform == "linux": - _verify_linux_glibc_floor(wheel, names, required_bins) + linux_elf_members = required_bins | native_extensions + _verify_linux_glibc_floor(wheel, names, linux_elf_members) if expected_arch is not None: - _verify_linux_bin_arch(wheel, names, required_bins, expected_arch) + _verify_linux_bin_arch(wheel, names, linux_elf_members, expected_arch) print(f"Verified wheel contents: {wheel}") From da732612cbe50e7448c48bfa4ac9f372702746bc Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Wed, 19 Aug 2026 15:03:19 +0000 Subject: [PATCH 52/52] feat: recommend a separate virtual environment for roar at init Installing roar into the workload's own environment has two costs, and the hint names both rather than dwelling on either. Both sets of requirements must resolve together, so roar's pins can collide with the project's. And roar's dependencies are loaded into the traced process and recorded alongside the project's -- measured on an `import requests` workload, the freeze carries nine packages belonging to roar. They cannot be separated afterwards: roar's copy of a package and the workload's are the same file at the same path, so nothing -- path, name, or dist metadata -- can attribute them. Subtracting by name once stripped the workload's own tqdm and typing-extensions (P0-28), which is why `roar_footprint_paths` abstains and the freeze over-includes instead. #287 tried to fix the second cost by snapshotting sys.modules at the end of bootstrap and subtracting it. That is measurably inert -- roar's dependencies load lazily *after* the boundary -- while risking a real false negative, so it was closed in favour of saying this plainly. The comparison is against the interpreter the WORKLOAD would use, not roar's own. That distinction is the whole check: under `uv tool` or pipx, roar runs from its own venv and so always sits inside its own sys.prefix, so comparing roar against itself reports every correctly isolated install as shared -- nagging exactly the users who took the advice. Resolution mirrors a shell's: active virtualenv, else conda env, else the first python on PATH; unresolvable means stay quiet. Verified live in both layouts rather than by assumption: roar copied into a project venv warns; roar in its own venv with a project venv active stays silent; and with no venv active, a tool install still stays silent because the workload would run the system python. Printed once at `roar init`, not per run: it is a property of how roar was installed, and a per-run warning is noise people learn to skip. It rides the existing hint machinery, so `roar config set hints.enabled false` already silences it. Both uv and pipx are offered, with install routes, since not everyone has uv. Seven tests over the real layouts (pip-into-venv, tool-install with and without an active venv, system install, conda), each verified to fail against the naive roar-vs-its-own-prefix comparison. Detection cannot fail the command; a cosmetic hint is never worth breaking `roar init` for. Co-Authored-By: Claude Opus 5 (1M context) --- roar/cli/commands/init.py | 58 ++++++++++++++++++++++++++++++ tests/unit/test_cli_init.py | 72 +++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/roar/cli/commands/init.py b/roar/cli/commands/init.py index 9e45ce1d..324179c8 100644 --- a/roar/cli/commands/init.py +++ b/roar/cli/commands/init.py @@ -4,7 +4,10 @@ Usage: roar init """ +import os +import shutil import sqlite3 as _sqlite3 +import sys from pathlib import Path import click @@ -357,6 +360,54 @@ def _print_version_header() -> None: print_brand_header("init") +def roar_shares_this_environment() -> bool: + """Whether roar is installed into the environment a workload would run in. + + Sharing one environment means roar's requirements and the project's have to + resolve together, and roar's dependencies are loaded into the traced process + and recorded alongside the project's. They cannot be told apart afterwards: + roar's copy of a package and the workload's are the same file at the same + path, so nothing -- path, name, or dist metadata -- can attribute them. + Subtracting by name once stripped the workload's own tqdm and + typing-extensions, so the freeze over-includes instead (see + ``roar_footprint_paths``). Better to recommend separate environments up + front than to discover either problem in a published record. + """ + try: + workload_prefix = _workload_interpreter_prefix() + if workload_prefix is None: + return False + return os.path.abspath(sys.prefix) == workload_prefix + except Exception: + # Never let a cosmetic hint break `roar init`. + return False + + +def _workload_interpreter_prefix() -> str | None: + """The prefix of the interpreter ``roar run python ...`` would use. + + Deliberately NOT ``sys.prefix``: that is *roar's* interpreter. Under a + ``uv tool`` or pipx install roar runs from its own venv, so roar always sits + under its own prefix and comparing the two would report every correctly + isolated install as shared -- nagging exactly the people who took the advice. + + Resolution mirrors what a shell would do: the active virtualenv or conda + env, else the first ``python`` on PATH. Returns None when no interpreter can + be resolved, which is treated as "say nothing". + """ + for env_var in ("VIRTUAL_ENV", "CONDA_PREFIX"): + value = os.environ.get(env_var) + if value: + return os.path.abspath(value) + + for name in ("python3", "python"): + found = shutil.which(name) + if found: + # /bin/python -> + return os.path.abspath(os.path.dirname(os.path.dirname(os.path.realpath(found)))) + return None + + def _maybe_print_init_hints(*, in_git_repo: bool, gitignore_action: str | None) -> None: """Print git-style `hint:` lines for next steps. Amber-colored to match git's hint convention. Suppressed in quiet/non-TTY contexts.""" @@ -384,6 +435,13 @@ def _maybe_print_init_hints(*, in_git_repo: bool, gitignore_action: str | None) hint() hint("Tracer auto-selects (eBPF → preload → ptrace). Switch with `roar tracer `;") hint("see all backends and readiness with `roar tracer`.") + if roar_shares_this_environment(): + hint() + hint("roar is installed in the same environment as your project. We recommend") + hint("running roar from its own virtual environment: it prevents version") + hint("conflicts and keeps them out of your lineage.") + hint(" uv tool install roar-cli # uv: https://astral.sh/uv") + hint(" pipx install roar-cli # pipx: sudo apt install pipx | brew install pipx") if in_git_repo: hint() hint("`roar run` requires a clean git tree — runs are tagged with the commit SHA.") diff --git a/tests/unit/test_cli_init.py b/tests/unit/test_cli_init.py index a89f2f22..a7e90a7f 100644 --- a/tests/unit/test_cli_init.py +++ b/tests/unit/test_cli_init.py @@ -301,3 +301,75 @@ def test_init_path_uses_target_repo_for_gitignore_updates(tmp_path: Path) -> Non assert caller_gitignore.read_text() == ".roar/\n" assert ".roar/" in target_gitignore.read_text().splitlines() assert (target_repo / ".roar").is_dir() + + +class TestSharedEnvironmentDetection: + """roar sharing the workload's environment means both sets of requirements + must resolve together, and roar's dependencies land in the freeze where + nothing can attribute them (P0-28). The comparison must be against the + interpreter the WORKLOAD would use, not roar's own: under `uv tool` or pipx + roar always sits inside its own prefix, so comparing roar to itself reports + every correctly isolated install as shared -- nagging exactly the users who + took the advice.""" + + def _prefixes(self, monkeypatch, *, roar_prefix, venv=None, conda=None, path_python=None): + from roar.cli.commands import init as init_module + + monkeypatch.setattr(init_module.sys, "prefix", roar_prefix, raising=False) + monkeypatch.delenv("VIRTUAL_ENV", raising=False) + monkeypatch.delenv("CONDA_PREFIX", raising=False) + if venv: + monkeypatch.setenv("VIRTUAL_ENV", venv) + if conda: + monkeypatch.setenv("CONDA_PREFIX", conda) + monkeypatch.setattr(init_module.shutil, "which", lambda _name: path_python, raising=False) + return init_module + + def test_pip_installed_into_the_active_project_venv_is_shared(self, monkeypatch): + init_module = self._prefixes(monkeypatch, roar_prefix="/proj/.venv", venv="/proj/.venv") + assert init_module.roar_shares_this_environment() is True + + def test_a_tool_install_alongside_an_active_project_venv_is_isolated(self, monkeypatch): + """The `uv tool` / pipx layout: roar runs from its own venv.""" + init_module = self._prefixes( + monkeypatch, roar_prefix="/home/u/.local/share/uv/tools/roar-cli", venv="/proj/.venv" + ) + assert init_module.roar_shares_this_environment() is False + + def test_a_tool_install_with_no_venv_active_is_isolated(self, monkeypatch): + """No venv: the workload would run the system python, which is not roar's.""" + init_module = self._prefixes( + monkeypatch, + roar_prefix="/home/u/.local/share/uv/tools/roar-cli", + path_python="/usr/bin/python3", + ) + assert init_module.roar_shares_this_environment() is False + + def test_a_system_install_with_no_venv_is_shared(self, monkeypatch): + init_module = self._prefixes( + monkeypatch, roar_prefix="/usr", path_python="/usr/bin/python3" + ) + assert init_module.roar_shares_this_environment() is True + + def test_a_conda_environment_is_honoured(self, monkeypatch): + init_module = self._prefixes( + monkeypatch, roar_prefix="/opt/conda/envs/proj", conda="/opt/conda/envs/proj" + ) + assert init_module.roar_shares_this_environment() is True + + def test_no_resolvable_interpreter_says_nothing(self, monkeypatch): + init_module = self._prefixes(monkeypatch, roar_prefix="/anything", path_python=None) + assert init_module.roar_shares_this_environment() is False + + def test_detection_never_breaks_init(self, monkeypatch): + """A cosmetic hint must not be able to fail `roar init`.""" + from roar.cli.commands import init as init_module + + def _boom(_name): + raise OSError("PATH exploded") + + monkeypatch.delenv("VIRTUAL_ENV", raising=False) + monkeypatch.delenv("CONDA_PREFIX", raising=False) + monkeypatch.setattr(init_module.shutil, "which", _boom, raising=False) + + assert init_module.roar_shares_this_environment() is False