diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 235e801c..67061ca6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,8 @@ jobs: rustup component add rust-src --toolchain nightly - name: Install bpf-linker - run: cargo install bpf-linker + # Keep CI reproducible; upgrade alongside its required LLVM toolchain. + run: cargo install bpf-linker --version 0.10.4 --locked - name: Build and stage Rust binaries uses: ./.github/actions/build-rust-binaries @@ -272,11 +273,32 @@ jobs: python -m roar run "$(python -c 'import sys; print(sys.executable)')" smoke.py - - name: Run tests (parallel) + - name: Run tracer / platform-dependent tests (parallel) + # macOS CI exists to cover the OS-specific surface: the DYLD preload + # tracer, sitecustomize/runtime injection, the native _hash_native + # extension, and the real `roar run` product path. The rest of the suite + # is OS-independent pure-Python logic already fully covered by the Linux + # `test` job (5 Python versions), so running all ~1180 tests here only + # adds slow-runner wall-clock (and timeouts) without adding signal. + # + # Select the platform-dependent trees by path rather than by marker: + # low-churn, self-documenting, and fails safe — Linux-only tracer + # regressions living inside these dirs carry their own + # skipif(platform != "Linux") and simply skip on macOS, and there is no + # per-file marker to forget to apply (which is how a macOS-only tracer + # test would otherwise be silently dropped). The two hashing-value files + # are kept so a macOS-specific _hash_native ABI/endianness regression + # still can't slip through, even though the hashing *logic* is covered + # on Linux. run: > pytest --tb=short -x -m "not glaas and not live_glaas and not ebpf and not large_pipeline" - --ignore=tests/backends/osmo - --ignore-glob=tests/backends/test_osmo*.py + tests/execution/runtime + tests/integration + tests/happy_path + tests/application/run + tests/backends/local/integration + tests/unit/test_hashing_backend.py + tests/unit/test_canonical_session_hash.py --ignore=tests/execution/runtime/test_sitecustomize_perf.py test-tracer-privileged: @@ -360,7 +382,7 @@ jobs: - name: Install build tools run: | python -m pip install --upgrade pip - pip install maturin + pip install 'maturin[zig]' - name: Download Rust binaries uses: actions/download-artifact@v4 @@ -378,7 +400,18 @@ jobs: - name: Build abi3 wheel env: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-deployment-target }} - run: maturin build --release --manifest-path rust/crates/artifact-hash-py/Cargo.toml --out dist + ROAR_WHEEL_PLATFORM: ${{ matrix.platform }} + run: | + args=( + build + --release + --manifest-path rust/crates/artifact-hash-py/Cargo.toml + --out dist + ) + if [[ "$ROAR_WHEEL_PLATFORM" == "linux" ]]; then + args+=(--zig --compatibility manylinux_2_17) + fi + maturin "${args[@]}" - name: Install uv run: | diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 7ec96a34..9090cea0 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -42,7 +42,8 @@ jobs: rustup component add rust-src --toolchain nightly - name: Install bpf-linker - run: cargo install bpf-linker + # Keep release builds aligned with the version validated in CI. + run: cargo install bpf-linker --version 0.10.4 --locked - name: Verify version matches release tag if: github.event_name == 'release' && matrix.arch == 'x86_64' @@ -164,7 +165,7 @@ jobs: - name: Install build tools run: | python -m pip install --upgrade pip - pip install maturin + pip install 'maturin[zig]' - name: Download Rust binaries artifact uses: actions/download-artifact@v4 @@ -184,7 +185,19 @@ jobs: # macOS floor on macOS jobs so the single abi3 build host doesn't drift it. env: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-deployment-target }} - run: maturin build --release --manifest-path rust/crates/artifact-hash-py/Cargo.toml --interpreter python --out dist + ROAR_WHEEL_PLATFORM: ${{ matrix.platform }} + run: | + args=( + build + --release + --manifest-path rust/crates/artifact-hash-py/Cargo.toml + --interpreter python + --out dist + ) + if [[ "$ROAR_WHEEL_PLATFORM" == "linux" ]]; then + args+=(--zig --compatibility manylinux_2_17) + fi + maturin "${args[@]}" - name: Verify wheel contains native extension and binaries env: diff --git a/.github/workflows/publish-testpypi.yml b/.github/workflows/publish-testpypi.yml index 7a81603f..a283cab3 100644 --- a/.github/workflows/publish-testpypi.yml +++ b/.github/workflows/publish-testpypi.yml @@ -35,7 +35,8 @@ jobs: rustup component add rust-src --toolchain nightly - name: Install bpf-linker - run: cargo install bpf-linker + # Keep release builds aligned with the version validated in CI. + run: cargo install bpf-linker --version 0.10.4 --locked - name: Build and stage Rust binaries uses: ./.github/actions/build-rust-binaries @@ -131,7 +132,7 @@ jobs: - name: Install build tools run: | python -m pip install --upgrade pip - pip install maturin + pip install 'maturin[zig]' - name: Download Rust binaries artifact uses: actions/download-artifact@v4 @@ -151,7 +152,19 @@ jobs: # macOS floor on macOS jobs so the single abi3 build host doesn't drift it. env: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-deployment-target }} - run: maturin build --release --manifest-path rust/crates/artifact-hash-py/Cargo.toml --interpreter python --out dist + ROAR_WHEEL_PLATFORM: ${{ matrix.platform }} + run: | + args=( + build + --release + --manifest-path rust/crates/artifact-hash-py/Cargo.toml + --interpreter python + --out dist + ) + if [[ "$ROAR_WHEEL_PLATFORM" == "linux" ]]; then + args+=(--zig --compatibility manylinux_2_17) + fi + maturin "${args[@]}" - name: Verify wheel contains native extension and binaries env: diff --git a/pyproject.toml b/pyproject.toml index 83f69f95..d1f63d0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "maturin" [project] name = "roar-cli" -version = "0.4.3" +version = "0.4.4" description = "Reproducibility and provenance tracker for ML training pipelines" authors = [ { name="TReqs Team", email="info@treqs.ai" } @@ -49,6 +49,7 @@ dependencies = [ "pydantic-settings>=2.0.0", "textual>=0.80", "tomli>=2.0.0; python_version < '3.11'", + "huggingface_hub>=0.20.0", # `roar put hf://` is a shipped path, not dev-only (P0-19) ] [project.urls] @@ -68,7 +69,6 @@ dev = [ "mypy>=1.13.0", "boto3>=1.28.0", "google-cloud-storage>=2.10.0", - "huggingface_hub>=0.20.0", ] [project.scripts] @@ -129,7 +129,7 @@ markers = [ "ray_contract: User-facing Ray contract tests using `roar run ray job submit ...`", "ray_diagnostic: Diagnostic Ray tests that intentionally inspect internal runtime details", ] -addopts = "-v --strict-markers -n auto --dist loadfile --ignore=tests/ebpf --ignore=tests/live_glaas --ignore=tests/benchmarks --ignore=tests/e2e --ignore=tests/integration/test_cli_startup.py --ignore=tests/execution/runtime/test_sitecustomize_perf.py --ignore-glob=tests/backends/*/e2e --ignore-glob=tests/backends/*/live" +addopts = "-v --strict-markers -n auto --dist loadfile --ignore=tests/ebpf --ignore=tests/live_glaas --ignore=tests/benchmarks --ignore=tests/e2e --ignore=tests/backends/osmo --ignore=tests/integration/test_cli_startup.py --ignore=tests/execution/runtime/test_sitecustomize_perf.py --ignore-glob=tests/backends/*/e2e --ignore-glob=tests/backends/*/live" timeout = 60 filterwarnings = [ "ignore::DeprecationWarning", diff --git a/roar/application/publish/collection.py b/roar/application/publish/collection.py index 7e6800f1..b02ade57 100644 --- a/roar/application/publish/collection.py +++ b/roar/application/publish/collection.py @@ -44,6 +44,11 @@ def collect_register_lineage( dry_run: bool = False, ) -> tuple[CollectedRegisterLineage | None, str | None]: """Collect local lineage for a resolved register target.""" + if target.kind == "active_session": + return _collect_active_session_lineage( + roar_dir=roar_dir, + lineage_collector=lineage_collector, + ) if target.kind == "step_reference": return _collect_step_lineage( step_reference=target.value, @@ -81,6 +86,29 @@ def collect_register_lineage( return None, f"Unsupported register target type: {target.kind}" +def _collect_active_session_lineage( + *, + roar_dir: Path, + lineage_collector: LineageCollector, +) -> tuple[CollectedRegisterLineage | None, str | None]: + with create_database_context(roar_dir) as db_ctx: + session = db_ctx.sessions.get_active() + if not session: + return None, "No active session. Run 'roar run' to create a session first." + session_id = int(session["id"]) + lineage = lineage_collector.collect_session(session_id, roar_dir) + + return ( + CollectedRegisterLineage( + lineage=lineage, + session_id=session_id, + artifact_hash="", + session_hash_override=None, + ), + None, + ) + + def _collect_step_lineage( *, step_reference: str, diff --git a/roar/application/publish/lineage_composites.py b/roar/application/publish/lineage_composites.py index 4e3ddd90..61714861 100644 --- a/roar/application/publish/lineage_composites.py +++ b/roar/application/publish/lineage_composites.py @@ -54,6 +54,7 @@ def preregister_lineage_composites_with_glaas( registration_errors: list[str], composite_builder: Any, logger: ILogger, + registration_session_id: str | None = None, ) -> list[dict[str, Any]]: """Prepare and preregister lineage composites before batch link resolution.""" payloads = build_lineage_composite_payloads( @@ -68,6 +69,7 @@ def preregister_lineage_composites_with_glaas( payloads=payloads, registration_errors=registration_errors, logger=logger, + registration_session_id=registration_session_id, ) diff --git a/roar/application/publish/put_composites.py b/roar/application/publish/put_composites.py index 7b48fd50..19d931b9 100644 --- a/roar/application/publish/put_composites.py +++ b/roar/application/publish/put_composites.py @@ -39,6 +39,7 @@ def preregister_put_lineage_composites_with_glaas( dataset_identifiers: list[dict[str, Any]] | None, composite_builder: Any, logger: ILogger, + registration_session_id: str | None = None, ) -> list[dict[str, Any]]: """Prepare and preregister lineage composites for the put workflow.""" payloads = build_put_lineage_composite_payloads( @@ -55,6 +56,7 @@ def preregister_put_lineage_composites_with_glaas( payloads=payloads, registration_errors=registration_errors, logger=logger, + registration_session_id=registration_session_id, ) @@ -164,6 +166,7 @@ def register_put_composites_with_glaas( registration_errors: list[str], dataset_identifiers: list[dict[str, Any]] | None, logger: ILogger, + registration_session_id: str | None = None, ) -> list[dict[str, Any]]: """Register generated composite artifacts with GLaaS and persist local state.""" composite_registrations: list[dict[str, Any]] = [] @@ -186,7 +189,14 @@ def register_put_composites_with_glaas( if metadata_json is not None: payload["metadata"] = metadata_json - response = resolved_remote_registry.register_composite_artifact(payload) + response = ( + resolved_remote_registry.client.register_composite_artifact_under_registration_session( + registration_session_id, + payload, + ) + if registration_session_id + else resolved_remote_registry.register_composite_artifact(payload) + ) result, error = parse_composite_registration_response(response) composite_registration: dict[str, Any] = { diff --git a/roar/application/publish/put_execution.py b/roar/application/publish/put_execution.py index 962906f9..907cc7dd 100644 --- a/roar/application/publish/put_execution.py +++ b/roar/application/publish/put_execution.py @@ -19,7 +19,7 @@ from ...application.publish.composites import build_publish_composite_results from ...application.publish.lineage import LineageCollector from ...application.publish.metadata import build_put_operation_metadata_json -from ...application.publish.put_preparation import PreparedPutExecution +from ...application.publish.put_preparation import DelegatedPutOperation, PreparedPutExecution from ...application.publish.registration import ( normalize_registration_hashes, normalize_registration_source_type, @@ -220,6 +220,7 @@ def put_prepared( session_hash = prepared.session_hash registration_session_id = prepared.registration_session_id registration_session_mode = prepared.registration_session_mode + registration_session_status = prepared.registration_session_status git_context = prepared.git_context resolved = prepared.resolved_sources destination_type = prepared.destination_type @@ -246,12 +247,31 @@ def put_prepared( would_upload=[PutDryRunItem(path=str(r.path), exists=r.exists) for r in resolved], ) + # Delegated broker sessions use a deterministic client-session id. If a + # previous attempt reached finalize but the caller lost the response, + # create/resume returns the closed session and its authoritative receipt. + # Treat that as a completed retry before hashing or uploading anything; + # closed registration-session capabilities cannot accept more staging and + # re-uploading large artifacts would be both wasteful and misleading. + if registration_session_id and registration_session_status == "closed": + self._logger.debug( + "Put publication already finalized for registration session %s", + registration_session_id, + ) + return PutResult( + success=True, + session_hash=session_hash, + session_url=prepared.session_url, + ) + # Process each file: hash, create artifact, upload uploads: list[_UploadedArtifact] = [] composite_registrations: list[dict[str, Any]] = [] lineage_composite_registrations: list[dict[str, Any]] = [] - with Spinner(f"Hashing {len(resolved)} file(s)..."): - hashes_by_path = self._hash_files_batch([source.path for source in resolved]) + hashes_by_path = prepared.source_hashes + if not hashes_by_path: + with Spinner(f"Hashing {len(resolved)} file(s)..."): + hashes_by_path = self._hash_files_batch([source.path for source in resolved]) # Uploads are the long pole of a put (multi-GB artifacts to S3/GCS); show a # live N/M + cumulative-bytes counter rather than a dead terminal. @@ -324,8 +344,11 @@ def put_prepared( ) # Collect lineage for all uploaded artifacts (merged) - collector = self._lineage_collector or LineageCollector() - lineage = collector.collect(artifact_hashes, self._roar_dir) + if prepared.lineage is not None: + lineage = prepared.lineage + else: + collector = self._lineage_collector or LineageCollector() + lineage = collector.collect(artifact_hashes, self._roar_dir) self._logger.debug( "Lineage collected: %d job(s), %d artifact(s)", len(lineage.jobs), @@ -340,6 +363,7 @@ def put_prepared( coordinator=coordinator, registration_session_id=registration_session_id, registration_session_mode=registration_session_mode, + delegated_put_operation=prepared.delegated_put_operation, session_id=session_id, fallback_session_hash=session_hash or "", git_context=git_context, @@ -600,6 +624,7 @@ def _put_prepared_with_registration_session( coordinator: RegistrationCoordinator, registration_session_id: str, registration_session_mode: str | None, + delegated_put_operation: DelegatedPutOperation | None, session_id: int, fallback_session_hash: str, git_context: GitContext, @@ -643,18 +668,30 @@ def _put_prepared_with_registration_session( timestamp=time.time(), ) - step_number = self._db.sessions.get_next_step_number(session_id) - job_id, job_uid = self._db.jobs.create( - command=command, - timestamp=time.time(), - session_id=session_id, - step_number=step_number, - metadata=provisional_metadata_json, - execution_backend="local", - execution_role="host", - job_type="put", - exit_code=0, + stable_put_job_uid = ( + delegated_put_operation.put_job_uid if delegated_put_operation is not None else None + ) + existing_put_job = ( + self._db.jobs.get_by_uid(stable_put_job_uid) if stable_put_job_uid else None ) + if existing_put_job is not None: + job_id = int(existing_put_job["id"]) + job_uid = str(existing_put_job["job_uid"]) + step_number = int(existing_put_job.get("step_number") or 0) + else: + step_number = self._db.sessions.get_next_step_number(session_id) + job_id, job_uid = self._db.jobs.create( + command=command, + timestamp=time.time(), + job_uid=stable_put_job_uid, + session_id=session_id, + step_number=step_number, + metadata=provisional_metadata_json, + execution_backend="local", + execution_role="host", + job_type="put", + exit_code=0, + ) self._logger.debug( "Put job created before registration-session finalize: id=%s, uid=%s, step=%d", job_id, @@ -675,19 +712,56 @@ def _put_prepared_with_registration_session( composite_builder=self._composite_builder, declared=declared, ) + uploaded_artifacts = self._build_uploaded_artifacts_for_registration( + uploads, + source_type, + ) + staged_artifacts = prepare_batch_registration_artifacts( + uploaded_artifacts + lineage.artifacts, + registration_session_id, + fallback_to_hash=True, + prefer_blake3_first=True, + ) put_job_registered = False put_job_links_succeeded = False with Spinner("Publishing lineage to GLaaS...") as spin: + spin.update("Staging lineage composites...") + lineage_composite_registrations = preregister_put_lineage_composites_with_glaas( + db_ctx=self._db, + glaas_client=client, + lineage_artifacts=lineage.artifacts, + session_hash=fallback_session_hash, + registration_errors=registration_errors, + dataset_identifiers=dataset_identifiers, + composite_builder=self._composite_builder, + logger=self._logger, + registration_session_id=registration_session_id, + ) + spin.update("Staging output composites...") + composite_registrations = register_put_composites_with_glaas( + db_ctx=self._db, + glaas_client=client, + composite_results=composite_results_for_linking, + registration_errors=registration_errors, + dataset_identifiers=dataset_identifiers, + logger=self._logger, + registration_session_id=registration_session_id, + ) spin.update("Staging lineage jobs and artifacts...") registration_result = coordinator.register_lineage_under_registration_session( registration_session_id=registration_session_id, git_context=git_context, jobs=remote_lineage_jobs, + artifacts=staged_artifacts, ) registration_errors.extend(registration_result.errors) - if registration_result.jobs_failed == 0 and registration_result.links_failed == 0: + if ( + registration_result.jobs_failed == 0 + and registration_result.links_failed == 0 + and not registration_errors + ): spin.update("Staging put job...") put_job_result = coordinator.job_service.create_job_under_registration_session( command=command, @@ -757,39 +831,6 @@ def _put_prepared_with_registration_session( session_hash = finalize_result.session_hash session_url = finalize_result.session_url - spin.update("Registering lineage composites...") - lineage_composite_registrations = ( - preregister_put_lineage_composites_with_glaas( - db_ctx=self._db, - glaas_client=client, - lineage_artifacts=lineage.artifacts, - session_hash=session_hash, - registration_errors=registration_errors, - dataset_identifiers=dataset_identifiers, - composite_builder=self._composite_builder, - logger=self._logger, - ) - ) - - composite_results = build_publish_composite_results( - resolved_sources=resolved, - hashes_by_path=hashes_by_path, - session_hash=session_hash, - source_type=composite_source_type, - additional_composite_roots=additional_composite_roots, - composite_builder=self._composite_builder, - declared=declared, - ) - spin.update("Registering output composites...") - composite_registrations = register_put_composites_with_glaas( - db_ctx=self._db, - glaas_client=client, - composite_results=composite_results, - registration_errors=registration_errors, - dataset_identifiers=dataset_identifiers, - logger=self._logger, - ) - metadata_json = build_put_operation_metadata_json( message=message, destination=self._destination, @@ -820,6 +861,7 @@ def _put_prepared_with_registration_session( remote_job_uid=remote_put_job_uid, registration_errors=registration_errors, uploads=uploads, + registration_session_id=registration_session_id, ) composite_result_items = [ @@ -1106,6 +1148,7 @@ def _sync_put_job_labels_with_glaas( remote_job_uid: str, registration_errors: list[str], uploads: list[_UploadedArtifact] | None = None, + registration_session_id: str | None = None, ) -> None: """Sync the local current label document for the publish-time put job and its published artifacts (carrying ``roar.distribution.url``).""" @@ -1122,6 +1165,7 @@ def _sync_put_job_labels_with_glaas( jobs=[{"id": job_id, "job_uid": job_uid, "remote_job_uid": remote_job_uid}], artifacts=artifacts, errors=registration_errors, + registration_session_id=registration_session_id, ) def _link_put_job_artifacts_with_glaas( diff --git a/roar/application/publish/put_preparation.py b/roar/application/publish/put_preparation.py index e6d04e69..7bd2d2bc 100644 --- a/roar/application/publish/put_preparation.py +++ b/roar/application/publish/put_preparation.py @@ -2,13 +2,23 @@ from __future__ import annotations +import hashlib +import json +import os +import secrets +import time +from collections.abc import Mapping from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Any from urllib.parse import urlparse +from sqlalchemy import text + +from ...core.interfaces.lineage import LineageData from ...core.interfaces.logger import ILogger from ...core.interfaces.registration import GitContext +from ...db.hashing import hash_files_blake3 from ...integrations.glaas import GlaasClient from ..git import resolve_roar_git_context from .datasets import ( @@ -23,6 +33,17 @@ from .source_resolution import ResolvedSource +@dataclass(frozen=True) +class DelegatedPutOperation: + """Durable local reservation for one broker-backed put operation.""" + + task_identity: str + session_id: int + ordinal: int + request_fingerprint: str + put_job_uid: str + + @dataclass(frozen=True) class PreparedPutExecution: """Application-prepared context for a put execution.""" @@ -35,10 +56,14 @@ class PreparedPutExecution: resolved_sources: list[ResolvedSource] destination_type: str composite_source_type: str | None + source_hashes: dict[str, str] = field(default_factory=dict) registration_session_id: str | None = None registration_session_mode: str | None = None + registration_session_status: str | None = None dataset_identifiers: list[dict[str, Any]] = field(default_factory=list) additional_composite_roots: dict[Path, list[ResolvedSource]] = field(default_factory=dict) + delegated_put_operation: DelegatedPutOperation | None = None + lineage: LineageData | None = None def prepare_put_execution( @@ -51,6 +76,7 @@ def prepare_put_execution( destination: str, git_commit: str | None, logger: ILogger, + operation_options: Mapping[str, Any] | None = None, ) -> PreparedPutExecution: """Resolve the local context needed to execute a put workflow.""" from .source_resolution import SourceResolver @@ -75,6 +101,81 @@ def prepare_put_execution( session_service=runtime.session_service, registration_coordinator=runtime_dict.get("registration_coordinator"), ) + resolver = SourceResolver( + repo_root=repo_root, + session_repo=db_ctx.sessions, + job_repo=db_ctx.jobs, + ) + resolved_sources = resolver.resolve(sources) + source_hashes = hash_files_blake3([source.path for source in resolved_sources]) + missing_hashes = [ + str(source.path) for source in resolved_sources if str(source.path) not in source_hashes + ] + if missing_hashes: + raise OSError(f"Failed to hash put source: {missing_hashes[0]}") + + operation_payload: dict[str, Any] = { + "destination": destination, + "git": { + "branch": git_context.branch, + "commit": git_context.commit, + "repo": git_context.repo, + }, + "local_session_hash": runtime.session_service.compute_session_hash( + roar_dir=str(roar_dir), + session_id=session_id, + ), + "local_session_id": session_id, + "options": dict(operation_options or {}), + "sources": sorted( + [ + { + "digest": source_hashes[str(source.path)], + "path": os.path.relpath(source.path.resolve(), repo_root.resolve()), + "relative_key": source.relative_key, + "size": source.path.stat().st_size, + } + for source in resolved_sources + ], + key=lambda source: (source["path"], source["relative_key"]), + ), + } + delegated_task_identity = _delegated_task_identity() + collected_lineage: LineageData | None = None + if delegated_task_identity is not None: + pending_put_job_uid = _pending_put_job_uid( + db_ctx, + delegated_task_identity, + session_id, + ) + from .lineage import LineageCollector + + collected_lineage = LineageCollector().collect( + sorted(set(source_hashes.values())), + roar_dir, + ) + collected_lineage = _without_lineage_job(collected_lineage, pending_put_job_uid) + operation_payload["lineage_revision"] = _collected_lineage_revision( + collected_lineage, + ) + request_fingerprint = _fingerprint(operation_payload) + delegated_put_operation = _reserve_delegated_put_operation( + db_ctx=db_ctx, + delegated_task_identity=delegated_task_identity, + session_id=session_id, + request_fingerprint=request_fingerprint, + ) + operation_fingerprint = ( + _fingerprint( + { + "ordinal": delegated_put_operation.ordinal, + "request_fingerprint": request_fingerprint, + } + ) + if delegated_put_operation is not None + else request_fingerprint + ) + publish_session = prepare_publish_session( remote_registry=remote_registry, roar_dir=roar_dir, @@ -82,14 +183,9 @@ def prepare_put_execution( git_context=git_context, logger=logger, register_with_glaas=True, + operation_kind="put", + operation_fingerprint=operation_fingerprint, ) - - resolver = SourceResolver( - repo_root=repo_root, - session_repo=db_ctx.sessions, - job_repo=db_ctx.jobs, - ) - resolved_sources = resolver.resolve(sources) dataset_identifiers = infer_publish_dataset_identifiers( repo_root=repo_root, source_specs=sources, @@ -112,14 +208,227 @@ def prepare_put_execution( git_context=git_context, registration_session_id=publish_session.registration_session_id, registration_session_mode=publish_session.registration_session_mode, + registration_session_status=( + publish_session.registration_session_status + if isinstance(publish_session.registration_session_status, str) + else None + ), resolved_sources=resolved_sources, destination_type=destination_type, composite_source_type=composite_source_type, + source_hashes=source_hashes, dataset_identifiers=dataset_identifiers, additional_composite_roots=additional_composite_roots, + delegated_put_operation=delegated_put_operation, + lineage=collected_lineage, ) +def complete_delegated_put_operation( + db_ctx: Any, + operation: DelegatedPutOperation | None, +) -> None: + """Mark a broker-backed put complete after its local and remote writes succeed.""" + if not isinstance(operation, DelegatedPutOperation): + return + + completed_at = time.time() + result = db_ctx.session.execute( + text( + """ + UPDATE delegated_put_operations + SET status = 'completed', updated_at = :completed_at, completed_at = :completed_at + WHERE task_identity = :task_identity + AND session_id = :session_id + AND ordinal = :ordinal + AND request_fingerprint = :request_fingerprint + AND status = 'pending' + """ + ), + { + "completed_at": completed_at, + "task_identity": operation.task_identity, + "session_id": operation.session_id, + "ordinal": operation.ordinal, + "request_fingerprint": operation.request_fingerprint, + }, + ) + if result.rowcount != 1: + raise RuntimeError("Delegated put operation reservation changed before completion") + db_ctx.commit() + + +def _delegated_task_identity() -> str | None: + values = [ + os.environ.get("ROAR_DELEGATED_JOB_ID", "").strip(), + os.environ.get("ROAR_DELEGATED_EXECUTION_ATTEMPT_ID", "").strip(), + os.environ.get("ROAR_DELEGATED_TASK_ID", "").strip(), + ] + if not any(values): + return None + if not all(values): + raise ValueError("Delegated publication task identity is incomplete") + return hashlib.sha256("\0".join(values).encode()).hexdigest() + + +def _pending_put_job_uid(db_ctx: Any, task_identity: str, session_id: int) -> str | None: + row = ( + db_ctx.session.execute( + text( + """ + SELECT put_job_uid + FROM delegated_put_operations + WHERE task_identity = :task_identity + AND session_id = :session_id + AND status = 'pending' + """ + ), + {"task_identity": task_identity, "session_id": session_id}, + ) + .mappings() + .one_or_none() + ) + return str(row["put_job_uid"]) if row and row["put_job_uid"] else None + + +def _without_lineage_job(lineage: LineageData, job_uid: str | None) -> LineageData: + if not job_uid: + return lineage + return LineageData( + jobs=[job for job in lineage.jobs if job.get("job_uid") != job_uid], + artifacts=lineage.artifacts, + artifact_hashes=lineage.artifact_hashes, + pipeline=lineage.pipeline, + ) + + +def _collected_lineage_revision(lineage: LineageData) -> str: + """Hash the exact target-rooted lineage snapshot passed to put execution.""" + jobs = sorted( + (_stable_json_value(job) for job in lineage.jobs), + key=lambda item: json.dumps(item, sort_keys=True, separators=(",", ":")), + ) + artifacts = sorted( + (_stable_json_value(artifact) for artifact in lineage.artifacts), + key=lambda item: json.dumps(item, sort_keys=True, separators=(",", ":")), + ) + return _fingerprint( + { + "jobs": jobs, + "artifacts": artifacts, + "artifact_hashes": sorted(lineage.artifact_hashes), + } + ) + + +def _stable_json_value(value: Any) -> Any: + if isinstance(value, Mapping): + return {str(key): _stable_json_value(child) for key, child in sorted(value.items())} + if isinstance(value, (list, tuple)): + return [_stable_json_value(child) for child in value] + if isinstance(value, set): + normalized = [_stable_json_value(child) for child in value] + return sorted(normalized, key=lambda child: json.dumps(child, sort_keys=True)) + if isinstance(value, Path): + return str(value) + if isinstance(value, bytes): + return value.hex() + return value + + +def _reserve_delegated_put_operation( + *, + db_ctx: Any, + delegated_task_identity: str | None, + session_id: int, + request_fingerprint: str, +) -> DelegatedPutOperation | None: + if delegated_task_identity is None: + return None + + now = time.time() + candidate_put_job_uid = f"delegated-put-{secrets.token_hex(12)}" + row = ( + db_ctx.session.execute( + text( + """ + INSERT INTO delegated_put_operations ( + task_identity, + session_id, + ordinal, + request_fingerprint, + put_job_uid, + status, + created_at, + updated_at, + completed_at + ) VALUES ( + :task_identity, + :session_id, + 1, + :request_fingerprint, + :put_job_uid, + 'pending', + :now, + :now, + NULL + ) + ON CONFLICT(task_identity, session_id) DO UPDATE SET + ordinal = CASE + WHEN delegated_put_operations.status = 'completed' + THEN delegated_put_operations.ordinal + 1 + ELSE delegated_put_operations.ordinal + END, + request_fingerprint = CASE + WHEN delegated_put_operations.status = 'completed' + THEN excluded.request_fingerprint + ELSE delegated_put_operations.request_fingerprint + END, + put_job_uid = CASE + WHEN delegated_put_operations.status = 'completed' + THEN excluded.put_job_uid + ELSE delegated_put_operations.put_job_uid + END, + status = 'pending', + updated_at = excluded.updated_at, + completed_at = NULL + WHERE delegated_put_operations.status = 'completed' + OR delegated_put_operations.request_fingerprint = excluded.request_fingerprint + RETURNING ordinal, request_fingerprint, put_job_uid + """ + ), + { + "task_identity": delegated_task_identity, + "session_id": session_id, + "request_fingerprint": request_fingerprint, + "put_job_uid": candidate_put_job_uid, + "now": now, + }, + ) + .mappings() + .one_or_none() + ) + if row is None: + raise ValueError( + "A different delegated put operation is already pending for this task; " + "retry the original command" + ) + db_ctx.commit() + return DelegatedPutOperation( + task_identity=delegated_task_identity, + session_id=session_id, + ordinal=int(row["ordinal"]), + request_fingerprint=str(row["request_fingerprint"]), + put_job_uid=str(row["put_job_uid"]), + ) + + +def _fingerprint(payload: Mapping[str, Any]) -> str: + return hashlib.sha256( + json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + def _destination_type(destination: str) -> str: parsed = urlparse(destination) return parsed.scheme or "local" diff --git a/roar/application/publish/register_execution.py b/roar/application/publish/register_execution.py index fb78ff77..a3775ba9 100644 --- a/roar/application/publish/register_execution.py +++ b/roar/application/publish/register_execution.py @@ -23,7 +23,10 @@ normalize_jobs_for_registration, order_jobs_for_registration, ) -from .remote_job_uids import prepare_jobs_for_remote_publication +from .remote_job_uids import ( + apply_remote_publication_job_uid_mapping, + prepare_jobs_for_remote_publication, +) from .secrets import ( detect_lineage_secrets, filter_git_context_secrets, @@ -223,6 +226,7 @@ def register_prepared_lineage( confirm_callback: Callable[[list[str]], bool] | None, prepared: PreparedRegisterExecution, composite_leaf_hashes: frozenset[str] = frozenset(), + view_edges_by_job: dict[str, list[Any]] | None = None, ) -> RegisterResult: """Register already-collected local lineage with GLaaS. @@ -241,6 +245,7 @@ def register_prepared_lineage( session_id = prepared.session_id registration_session_id = prepared.registration_session_id registration_session_mode = prepared.registration_session_mode + registration_session_status = prepared.registration_session_status omit_filter = self.omit_filter detected_secrets: list[str] = [] @@ -292,11 +297,36 @@ def register_prepared_lineage( registration_jobs = order_jobs_for_registration( normalize_jobs_for_registration(lineage.jobs) ) - remote_registration_jobs = ( - prepare_jobs_for_remote_publication(registration_jobs, session_hash) - if registration_session_id - else registration_jobs + closed_remote_uid_mapping: dict[str, str] = {} + if registration_session_id and registration_session_status == "closed" and session_id: + with create_database_context(roar_dir) as db_ctx: + closed_remote_uid_mapping = load_glaas_publication_job_mapping( + db_ctx=db_ctx, + session_id=session_id, + ) + if registration_session_id and closed_remote_uid_mapping: + remote_registration_jobs = apply_remote_publication_job_uid_mapping( + registration_jobs, + closed_remote_uid_mapping, + ) + elif registration_session_id: + remote_registration_jobs = prepare_jobs_for_remote_publication( + registration_jobs, + session_hash, + ) + else: + remote_registration_jobs = registration_jobs + missing_closed_remote_uid_mapping = bool( + registration_session_id + and registration_session_status == "closed" + and session_id + and not closed_remote_uid_mapping ) + if registration_session_id and view_edges_by_job: + for job in remote_registration_jobs: + local_job_uid = job.get("job_uid") + if isinstance(local_job_uid, str) and local_job_uid in view_edges_by_job: + job["_view_edges"] = view_edges_by_job[local_job_uid] if dry_run: return RegisterResult( @@ -319,15 +349,90 @@ def register_prepared_lineage( composite_registrations: list[dict[str, Any]] = [] registration_errors: list[str] = [] + if missing_closed_remote_uid_mapping: + registration_errors.append( + "Closed publication is missing its persisted remote job identity mapping" + ) finalized_session_hash = session_hash finalized_session_url = prepared.session_url finalize_failed = False already_registered = False + if registration_session_id and registration_session_status == "closed": + from ...core.interfaces.registration import BatchRegistrationResult + + already_registered = True + batch_result = BatchRegistrationResult( + session_registered=True, + jobs_created=0, + jobs_existing=len(remote_registration_jobs), + jobs_failed=0, + artifacts_registered=len(lineage.artifacts), + artifacts_failed=0, + links_created=0, + links_failed=0, + errors=[], + ) + if session_id is not None and not registration_errors: + with create_database_context(roar_dir) as db_ctx: + batch_result.labels_synced = sync_publish_labels( + glaas_client=self.glaas_client, + db_ctx=db_ctx, + session_id=session_id, + session_hash=finalized_session_hash, + jobs=remote_registration_jobs, + artifacts=label_artifacts, + errors=registration_errors, + registration_session_id=registration_session_id, + ) + success = not registration_errors + if success and session_id is not None: + with create_database_context(roar_dir) as db_ctx: + persist_glaas_publication_mapping( + db_ctx=db_ctx, + session_id=session_id, + prepared_session_hash=session_hash, + finalized_session_hash=finalized_session_hash, + jobs=remote_registration_jobs, + ) + mark_lineage_synced( + db_ctx=db_ctx, + session_id=session_id, + jobs=lineage.jobs, + artifacts=lineage.artifacts, + ) + return RegisterResult( + success=success, + session_hash=finalized_session_hash, + session_url=finalized_session_url, + artifact_hash=artifact_hash, + jobs_registered=0, + jobs_existing=len(remote_registration_jobs), + artifacts_registered=len(lineage.artifacts), + links_created=0, + labels_synced=batch_result.labels_synced, + already_registered=already_registered, + error="; ".join(registration_errors) if registration_errors else None, + secrets_detected=detected_secrets, + secrets_redacted=bool(detected_secrets), + ) with Spinner("Publishing lineage to GLaaS...") as spin: refresh_job_artifact_references(lineage.jobs, lineage.artifacts) if registration_session_id: spin.update("Staging jobs and artifacts...") + if has_lineage_composites(lineage.artifacts): + spin.update("Staging composite artifacts...") + with create_database_context(roar_dir) as db_ctx: + composite_registrations = preregister_lineage_composites_with_glaas( + glaas_client=self.glaas_client, + db_ctx=db_ctx, + lineage_artifacts=lineage.artifacts, + session_hash=session_hash, + registration_errors=registration_errors, + composite_builder=self.composite_builder, + logger=self._logger, + registration_session_id=registration_session_id, + ) staged_artifacts = prepare_batch_registration_artifacts( lineage.artifacts, registration_session_id, # placeholder; client strips it before send @@ -349,6 +454,24 @@ def register_prepared_lineage( already_registered = True finalized_session_hash = batch_result.already_registered_session_hash finalized_session_url = None + if batch_result.existing_binding_prepared: + finalize_result = ( + self.coordinator.session_service.finalize_registration_session( + registration_session_id=registration_session_id, + git_context=git_context, + ) + ) + if not finalize_result.success: + registration_errors.append( + "Existing publication binding finalize failed: " + f"{finalize_result.error}" + ) + elif finalize_result.session_hash != finalized_session_hash: + registration_errors.append( + "Existing publication binding returned a different lineage hash" + ) + else: + finalized_session_url = finalize_result.session_url if session_id is not None: with create_database_context(roar_dir) as db_ctx: batch_result.labels_synced = sync_publish_labels( @@ -359,6 +482,11 @@ def register_prepared_lineage( jobs=remote_registration_jobs, artifacts=label_artifacts, errors=registration_errors, + registration_session_id=( + registration_session_id + if batch_result.existing_binding_prepared + else None + ), ) elif batch_result.jobs_failed == 0 and batch_result.links_failed == 0: spin.update("Finalizing lineage...") @@ -382,41 +510,7 @@ def register_prepared_lineage( finalized_session_hash = finalize_result.session_hash finalized_session_url = finalize_result.session_url - if has_lineage_composites(lineage.artifacts): - spin.update("Registering composite artifacts...") - try: - with create_database_context(roar_dir) as db_ctx: - composite_registrations = ( - preregister_lineage_composites_with_glaas( - glaas_client=self.glaas_client, - db_ctx=db_ctx, - lineage_artifacts=lineage.artifacts, - session_hash=finalized_session_hash, - registration_errors=registration_errors, - composite_builder=self.composite_builder, - logger=self._logger, - ) - ) - if session_id is not None: - batch_result.labels_synced = sync_publish_labels( - glaas_client=self.glaas_client, - db_ctx=db_ctx, - session_id=session_id, - session_hash=finalized_session_hash, - jobs=remote_registration_jobs, - artifacts=label_artifacts, - errors=registration_errors, - ) - except Exception as e: - return RegisterResult( - success=False, - session_hash=finalized_session_hash, - artifact_hash=artifact_hash, - error=f"Composite artifact registration failed: {e}", - secrets_detected=detected_secrets, - secrets_redacted=bool(detected_secrets), - ) - elif session_id is not None: + if session_id is not None: with create_database_context(roar_dir) as db_ctx: batch_result.labels_synced = sync_publish_labels( glaas_client=self.glaas_client, @@ -426,6 +520,7 @@ def register_prepared_lineage( jobs=remote_registration_jobs, artifacts=label_artifacts, errors=registration_errors, + registration_session_id=registration_session_id, ) else: if has_lineage_composites(lineage.artifacts): @@ -506,7 +601,11 @@ def register_prepared_lineage( total_artifacts_registered = batch_result.artifacts_registered + composite_registered success = ( - batch_result.jobs_failed == 0 and total_artifacts_failed == 0 and not finalize_failed + batch_result.jobs_failed == 0 + and batch_result.links_failed == 0 + and total_artifacts_failed == 0 + and not finalize_failed + and not registration_errors ) if success and session_id is not None: try: @@ -614,6 +713,28 @@ def persist_glaas_publication_mapping( ) +def load_glaas_publication_job_mapping( + *, + db_ctx: Any, + session_id: int, +) -> dict[str, str]: + """Load the authoritative local-to-remote job mapping for label refresh.""" + session = db_ctx.sessions.get(session_id) + if not isinstance(session, dict): + return {} + metadata = _load_session_metadata(session.get("metadata")) + jobs = (((metadata.get("roar") or {}).get("remote_publication") or {}).get("glaas") or {}).get( + "jobs" + ) + if not isinstance(jobs, dict): + return {} + return { + str(local_uid): str(remote_uid) + for local_uid, remote_uid in jobs.items() + if isinstance(remote_uid, str) and remote_uid + } + + def _load_session_metadata(raw_metadata: Any) -> dict[str, Any]: if isinstance(raw_metadata, dict): return dict(raw_metadata) diff --git a/roar/application/publish/register_preparation.py b/roar/application/publish/register_preparation.py index 8c66c19c..2fe9ae34 100644 --- a/roar/application/publish/register_preparation.py +++ b/roar/application/publish/register_preparation.py @@ -28,6 +28,7 @@ class PreparedRegisterExecution: git_tag_repo_root: Path | None registration_session_id: str | None = None registration_session_mode: str | None = None + registration_session_status: str | None = None def prepare_register_execution( @@ -108,4 +109,9 @@ def prepare_register_execution( git_tag_repo_root=git_tag_repo_root, registration_session_id=publish_session.registration_session_id, registration_session_mode=publish_session.registration_session_mode, + registration_session_status=( + publish_session.registration_session_status + if isinstance(publish_session.registration_session_status, str) + else None + ), ) diff --git a/roar/application/publish/registration.py b/roar/application/publish/registration.py index 381d8001..0952bb53 100644 --- a/roar/application/publish/registration.py +++ b/roar/application/publish/registration.py @@ -414,6 +414,7 @@ def preregister_lineage_composites( payloads: list[CompositeRegistrationCandidate], registration_errors: list[str], logger: ILogger, + registration_session_id: str | None = None, ) -> list[dict[str, Any]]: """Register lineage composites before the main link phase.""" registrations: list[dict[str, Any]] = [] @@ -423,7 +424,14 @@ def preregister_lineage_composites( ) for item in payloads: - response = resolved_remote_registry.register_composite_artifact(item.payload) + response = ( + resolved_remote_registry.client.register_composite_artifact_under_registration_session( + registration_session_id, + item.payload, + ) + if registration_session_id + else resolved_remote_registry.register_composite_artifact(item.payload) + ) result, error = parse_composite_registration_response(response) registration: dict[str, Any] = { @@ -468,6 +476,7 @@ def sync_publish_labels( jobs: list[dict[str, Any]], artifacts: list[dict[str, Any]], errors: list[str] | None = None, + registration_session_id: str | None = None, ) -> int: """Sync current local labels for published entities to GLaaS. @@ -490,7 +499,15 @@ def sync_publish_labels( glaas_client=glaas_client, ) - _label_result, label_error = resolved_remote_registry.sync_labels(payloads) + if registration_session_id: + _label_result, label_error = ( + resolved_remote_registry.client.sync_labels_under_registration_session( + registration_session_id, + payloads, + ) + ) + else: + _label_result, label_error = resolved_remote_registry.sync_labels(payloads) if label_error: if errors is not None: errors.append(f"Label sync failed: {label_error}") diff --git a/roar/application/publish/remote_job_uids.py b/roar/application/publish/remote_job_uids.py index 296cb0d1..9684a4b1 100644 --- a/roar/application/publish/remote_job_uids.py +++ b/roar/application/publish/remote_job_uids.py @@ -48,3 +48,24 @@ def prepare_jobs_for_remote_publication( prepared_jobs.append(prepared) return prepared_jobs + + +def apply_remote_publication_job_uid_mapping( + jobs: list[dict[str, Any]], + remote_uid_by_local_uid: dict[str, str], +) -> list[dict[str, Any]]: + """Apply the authoritative mapping persisted by a completed publication.""" + prepared_jobs: list[dict[str, Any]] = [] + for job in jobs: + prepared = dict(job) + local_job_uid = prepared.get("job_uid") + if isinstance(local_job_uid, str) and local_job_uid: + remote_job_uid = remote_uid_by_local_uid.get(local_job_uid) + if remote_job_uid: + prepared["remote_job_uid"] = remote_job_uid + + parent_job_uid = prepared.get("parent_job_uid") + if isinstance(parent_job_uid, str) and parent_job_uid: + prepared["remote_parent_job_uid"] = remote_uid_by_local_uid.get(parent_job_uid) + prepared_jobs.append(prepared) + return prepared_jobs diff --git a/roar/application/publish/requests.py b/roar/application/publish/requests.py index ce9eda2c..50228250 100644 --- a/roar/application/publish/requests.py +++ b/roar/application/publish/requests.py @@ -11,7 +11,9 @@ class RegisterLineageRequest: """Application request for `roar register`.""" - target: str + # None means the whole active session. Keep that intent explicit instead + # of freezing a pre-bootstrap canonical hash in the CLI. + target: str | None roar_dir: Path cwd: Path dry_run: bool = False diff --git a/roar/application/publish/service.py b/roar/application/publish/service.py index 7b1363d7..8fe9264e 100644 --- a/roar/application/publish/service.py +++ b/roar/application/publish/service.py @@ -21,6 +21,7 @@ PutUploadedFile, RegisterLineageResponse, ) +from .targets import ResolvedRegisterTarget if TYPE_CHECKING: from ...db.query_context import QueryDatabaseContext @@ -98,6 +99,13 @@ def prepare_put_execution(*args: Any, **kwargs: Any) -> Any: return _prepare_put_execution(*args, **kwargs) +def complete_delegated_put_operation(*args: Any, **kwargs: Any) -> Any: + """Durably close a delegated put reservation after publication succeeds.""" + from .put_preparation import complete_delegated_put_operation as _complete + + return _complete(*args, **kwargs) + + def prepare_register_execution(*args: Any, **kwargs: Any) -> Any: """Load register preparation only when register runs.""" from .register_preparation import ( @@ -566,10 +574,14 @@ def register_lineage_target(request: RegisterLineageRequest) -> RegisterLineageR ) try: - resolved_target = resolve_register_lineage_target( - request.target, - cwd=request.cwd, - roar_dir=request.roar_dir, + resolved_target = ( + ResolvedRegisterTarget(kind="active_session", value="") + if request.target is None + else resolve_register_lineage_target( + request.target, + cwd=request.cwd, + roar_dir=request.roar_dir, + ) ) runtime_kwargs: dict[str, Any] = { "start_dir": str(request.cwd), @@ -738,6 +750,7 @@ def register_lineage_target(request: RegisterLineageRequest) -> RegisterLineageR confirm_callback=request.confirm_callback, prepared=prepared, composite_leaf_hashes=composite_leaf_hashes, + view_edges_by_job=view_edges_by_job, ) # Push the consumes view edges now that the jobs + the anchor composite are @@ -745,7 +758,7 @@ def register_lineage_target(request: RegisterLineageRequest) -> RegisterLineageR # under publication-scoped *remote* UIDs; translate via the mapping registration # persisted to the session metadata. Best-effort: never fails an otherwise- # successful registration. - if result.success and view_edges_by_job: + if result.success and view_edges_by_job and not prepared.registration_session_id: remote_uid_by_local = _load_remote_job_uid_mapping( roar_dir=request.roar_dir, session_id=collected_lineage.session_id ) @@ -881,6 +894,14 @@ def put_artifacts(request: PutRequest) -> PutResponse: destination=request.destination, git_commit=git_commit, logger=logger, + operation_options={ + "anonymous": request.anonymous, + "as_dataset": request.as_dataset, + "message": request.message, + "no_tag": request.no_tag, + "public": request.public, + "step_name": request.step_name, + }, ) # Reproducibility facts for the receipt: same commit-span source as @@ -916,6 +937,9 @@ def put_artifacts(request: PutRequest) -> PutResponse: ), ) + if result.success: + complete_delegated_put_operation(db_ctx, prepared.delegated_put_operation) + # Apply step name label if provided. if request.step_name and result.success and result.job_id: with contextlib.suppress(Exception): diff --git a/roar/application/publish/session.py b/roar/application/publish/session.py index 1ee28699..7a1b70f5 100644 --- a/roar/application/publish/session.py +++ b/roar/application/publish/session.py @@ -2,7 +2,9 @@ from __future__ import annotations +import hashlib import json +import os from dataclasses import dataclass from pathlib import Path from typing import Any, Protocol @@ -43,6 +45,54 @@ class PreparedPublishSession: session_url: str | None = None registration_session_id: str | None = None registration_session_mode: str | None = None + registration_session_status: str | None = None + + +def delegated_client_session_id( + *, + operation_kind: str, + operation_fingerprint: str, +) -> str | None: + """Return a stable retry key for one publication operation in a TReqs task.""" + identity = [ + os.environ.get("ROAR_DELEGATED_JOB_ID"), + os.environ.get("ROAR_DELEGATED_EXECUTION_ATTEMPT_ID"), + os.environ.get("ROAR_DELEGATED_TASK_ID"), + ] + if not all(identity): + return None + if not operation_kind or not operation_fingerprint: + raise ValueError("Delegated publication requires an operation identity") + digest = hashlib.sha256( + "\0".join( + [ + *(str(value) for value in identity), + operation_kind, + operation_fingerprint, + ] + ).encode() + ).hexdigest() + return f"roar-delegated-v2-{digest}" + + +def _dedup_edges_by_hash(edges: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Collapse edges to one per CONTENT hash — matching glaas, which keys job + inputs/outputs on ``(job_id, artifact_hash)`` and so drops duplicate-path edges + for the same bytes (timm's ``os.link`` last/best/checkpoint = one inode, three + names). Keeps the lexicographically-smallest path so the surviving edge is + deterministic across roar and glaas. Edges without a resolvable hash are + dropped. A no-op when a job has no byte-identical edges — i.e. every currently + passing row — so existing session hashes are unchanged. P0-22. + """ + by_hash: dict[str, dict[str, Any]] = {} + for edge in edges: + digest = _canonical_artifact_hash(edge) + if not digest: + continue + current = by_hash.get(digest) + if current is None or str(edge.get("path") or "") < str(current.get("path") or ""): + by_hash[digest] = edge + return list(by_hash.values()) def build_canonical_session_payload( @@ -63,7 +113,7 @@ def build_canonical_session_payload( "hash": _canonical_artifact_hash(artifact), "path": artifact.get("path"), } - for artifact in job.get("_inputs", []) + for artifact in _dedup_edges_by_hash(job.get("_inputs", [])) ], key=lambda artifact: ( str(artifact.get("hash") or ""), @@ -76,7 +126,7 @@ def build_canonical_session_payload( "hash": _canonical_artifact_hash(artifact), "path": artifact.get("path"), } - for artifact in job.get("_outputs", []) + for artifact in _dedup_edges_by_hash(job.get("_outputs", [])) ], key=lambda artifact: ( str(artifact.get("hash") or ""), @@ -110,11 +160,19 @@ def build_git_context_from_lineage(lineage: LineageData) -> GitContext: def build_staged_lineage_counts(jobs: list[dict[str, Any]]) -> dict[str, int]: - """Build lightweight finalize expectations for staged registration-session lineage.""" + """Finalize expectations for staged registration-session lineage. + + Counts DISTINCT content hashes per job — matching glaas, which stores job + edges keyed on ``(job_id, artifact_hash)`` and so collapses byte-identical + outputs written to several names (timm ``os.link`` last/best/checkpoint) to one + row. Asserting the raw per-path count made finalize 400 with "Staged lineage + counts did not match" (P0-22). A no-op for every currently-passing row (no + duplicate edges → distinct == path). + """ return { "jobs": len(jobs), - "inputs": sum(len(job.get("_inputs", [])) for job in jobs), - "outputs": sum(len(job.get("_outputs", [])) for job in jobs), + "inputs": sum(len(_dedup_edges_by_hash(job.get("_inputs", []))) for job in jobs), + "outputs": sum(len(_dedup_edges_by_hash(job.get("_outputs", []))) for job in jobs), } @@ -304,6 +362,8 @@ def prepare_publish_session( session_hash_override: str | None = None, lineage: LineageData | None = None, creator_identity: str | None = None, + operation_kind: str = "register", + operation_fingerprint: str | None = None, ) -> PreparedPublishSession: """Compute and optionally register the publish session.""" resolved_remote_registry = coerce_remote_registry( @@ -355,10 +415,14 @@ def prepare_publish_session( publish_auth = resolved_remote_registry.publish_auth access_token = getattr(publish_auth, "access_token", None) ssh_auth_available = getattr(publish_auth, "ssh_auth_available", False) + delegated_auth_available = getattr(publish_auth, "delegated_auth_available", False) scope_request = getattr(publish_auth, "scope_request", None) has_access_token = isinstance(access_token, str) and bool(access_token.strip()) has_ssh_auth = ssh_auth_available if isinstance(ssh_auth_available, bool) else False + has_delegated_auth = ( + delegated_auth_available if isinstance(delegated_auth_available, bool) else False + ) anonymous_public_capable = ( scope_request is None @@ -388,7 +452,7 @@ def prepare_publish_session( ) should_use_registration_sessions = ( - has_access_token or has_ssh_auth or supports_anonymous_public_path + has_access_token or has_ssh_auth or has_delegated_auth or supports_anonymous_public_path ) if should_use_registration_sessions: @@ -398,7 +462,14 @@ def prepare_publish_session( f" (mode={registration_session_mode})" if registration_session_mode else "", ) session_result = resolved_session_service.create_registration_session( - client_session_id=None, + client_session_id=( + delegated_client_session_id( + operation_kind=operation_kind, + operation_fingerprint=operation_fingerprint or session_hash, + ) + if has_delegated_auth + else None + ), mode=registration_session_mode, ) if not session_result.success: @@ -409,11 +480,29 @@ def prepare_publish_session( "Registration session ready: %s", session_result.registration_session_id, ) + if session_result.status == "closed" and session_result.registration_session_id: + finalized = resolved_session_service.finalize_registration_session( + registration_session_id=session_result.registration_session_id, + git_context=git_context, + ) + if not finalized.success: + raise ValueError( + "Closed registration session could not return its publication receipt: " + f"{finalized.error}" + ) + return PreparedPublishSession( + session_hash=finalized.session_hash, + session_url=finalized.session_url, + registration_session_id=session_result.registration_session_id, + registration_session_mode=session_result.registration_session_mode, + registration_session_status="closed", + ) return PreparedPublishSession( session_hash=session_hash, session_url=None, registration_session_id=session_result.registration_session_id, registration_session_mode=session_result.registration_session_mode, + registration_session_status=session_result.status, ) logger.debug("Registering session with GLaaS") diff --git a/roar/application/reproduce/requests.py b/roar/application/reproduce/requests.py index 76841191..dee05543 100644 --- a/roar/application/reproduce/requests.py +++ b/roar/application/reproduce/requests.py @@ -20,6 +20,11 @@ class ReproduceRequest: package_sync: bool = False list_requirements: bool = False out_path: str | None = None + # Write the recorded pip pins to a requirements.txt (for debugging a failed + # install) instead of previewing/running; None means don't export. + export_requirements: str | None = None + # Per-step wall-clock timeout in seconds for --run; None means no timeout. + step_timeout: int | None = None # Skip publish (`roar put`) steps — for third-party reproduction that # rebuilds the artifact without re-publishing to the owner's destination. no_puts: bool = False diff --git a/roar/application/reproduce/service.py b/roar/application/reproduce/service.py index 923b0e87..e4854986 100644 --- a/roar/application/reproduce/service.py +++ b/roar/application/reproduce/service.py @@ -91,6 +91,10 @@ def reproduce_artifact( hash_prefix=request.hash_prefix, ) + if request.export_requirements: + _export_pip_requirements(pipeline, request.export_requirements, output) + return + if not request.run_pipeline: _render_preview_summary( preview, @@ -163,7 +167,9 @@ def reproduce_artifact( return with _reproduction_session(environment.repo_dir, output): - steps_run, steps_total = PipelineExecutor(presenter=output).execute( + steps_run, steps_total = PipelineExecutor( + presenter=output, step_timeout=request.step_timeout + ).execute( pipeline, environment, request.auto_confirm, @@ -483,6 +489,28 @@ def build_reproduction_script( return "\n".join(lines) +def _export_pip_requirements(pipeline: PipelineInfo, path: str, output: IPresenter) -> None: + """Write the recorded pip pins to a requirements.txt for offline debugging. + + Complements ``--script`` (which emits the reproduction shell) by emitting the + *packages* in a pip-native form the user can try directly: + ``pip install --dry-run -r `` shows exactly which pins don't resolve + (yanked, private, or on an extra index). No ``uv`` assumption. + """ + summary = PipelineMetadataParser().summarize_requirements( + pipeline.build_steps, pipeline.run_steps + ) + target = pipeline.artifact_hash or pipeline.session_hash or "" + header = [ + f"# roar reproduce — recorded pip pins for {target[:12]}", + "# Try: pip install --dry-run -r (shows which pins do not resolve)", + "# NOTE: the recorded --index-url/--extra-index-url is not replayed here yet,", + "# so a pin published only on a custom index reads as 'not found'.", + ] + Path(path).write_text("\n".join([*header, *sorted(summary.pip)]) + "\n", encoding="utf-8") + output.print(f"Wrote {len(summary.pip)} pip requirement(s) to {path}") + + def build_preview_summary( pipeline: PipelineInfo, *, diff --git a/roar/application/reproducibility/report.py b/roar/application/reproducibility/report.py index 0665a1d8..2ddb655d 100644 --- a/roar/application/reproducibility/report.py +++ b/roar/application/reproducibility/report.py @@ -180,8 +180,11 @@ def runtime_captured(pipeline) -> bool: from ...execution.reproduction.pipeline_metadata import PipelineMetadataParser try: - runtime = PipelineMetadataParser().first_runtime(pipeline.build_steps, pipeline.run_steps) - return bool((runtime.get("python") or {}).get("version")) + parser = PipelineMetadataParser() + runtime = parser.first_runtime(pipeline.build_steps, pipeline.run_steps) + return bool( + (runtime.get("python") or {}).get("version") + ) and parser.python_capture_complete(pipeline.build_steps, pipeline.run_steps) except Exception: return False diff --git a/roar/cli/commands/init.py b/roar/cli/commands/init.py index e0645a66..324179c8 100644 --- a/roar/cli/commands/init.py +++ b/roar/cli/commands/init.py @@ -4,7 +4,10 @@ Usage: roar init """ +import os +import shutil import sqlite3 as _sqlite3 +import sys from pathlib import Path import click @@ -81,7 +84,10 @@ "GITHUB_TOKEN", "DATABASE_URL", "AWS_SECRET_ACCESS_KEY", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", ] +# (HF token *values* are also caught unconditionally by roar's built-in patterns.) [registration.omit.allowlist] # Regex patterns that should NOT be redacted (reduce false positives) @@ -354,6 +360,54 @@ def _print_version_header() -> None: print_brand_header("init") +def roar_shares_this_environment() -> bool: + """Whether roar is installed into the environment a workload would run in. + + Sharing one environment means roar's requirements and the project's have to + resolve together, and roar's dependencies are loaded into the traced process + and recorded alongside the project's. They cannot be told apart afterwards: + roar's copy of a package and the workload's are the same file at the same + path, so nothing -- path, name, or dist metadata -- can attribute them. + Subtracting by name once stripped the workload's own tqdm and + typing-extensions, so the freeze over-includes instead (see + ``roar_footprint_paths``). Better to recommend separate environments up + front than to discover either problem in a published record. + """ + try: + workload_prefix = _workload_interpreter_prefix() + if workload_prefix is None: + return False + return os.path.abspath(sys.prefix) == workload_prefix + except Exception: + # Never let a cosmetic hint break `roar init`. + return False + + +def _workload_interpreter_prefix() -> str | None: + """The prefix of the interpreter ``roar run python ...`` would use. + + Deliberately NOT ``sys.prefix``: that is *roar's* interpreter. Under a + ``uv tool`` or pipx install roar runs from its own venv, so roar always sits + under its own prefix and comparing the two would report every correctly + isolated install as shared -- nagging exactly the people who took the advice. + + Resolution mirrors what a shell would do: the active virtualenv or conda + env, else the first ``python`` on PATH. Returns None when no interpreter can + be resolved, which is treated as "say nothing". + """ + for env_var in ("VIRTUAL_ENV", "CONDA_PREFIX"): + value = os.environ.get(env_var) + if value: + return os.path.abspath(value) + + for name in ("python3", "python"): + found = shutil.which(name) + if found: + # /bin/python -> + return os.path.abspath(os.path.dirname(os.path.dirname(os.path.realpath(found)))) + return None + + def _maybe_print_init_hints(*, in_git_repo: bool, gitignore_action: str | None) -> None: """Print git-style `hint:` lines for next steps. Amber-colored to match git's hint convention. Suppressed in quiet/non-TTY contexts.""" @@ -381,6 +435,13 @@ def _maybe_print_init_hints(*, in_git_repo: bool, gitignore_action: str | None) hint() hint("Tracer auto-selects (eBPF → preload → ptrace). Switch with `roar tracer `;") hint("see all backends and readiness with `roar tracer`.") + if roar_shares_this_environment(): + hint() + hint("roar is installed in the same environment as your project. We recommend") + hint("running roar from its own virtual environment: it prevents version") + hint("conflicts and keeps them out of your lineage.") + hint(" uv tool install roar-cli # uv: https://astral.sh/uv") + hint(" pipx install roar-cli # pipx: sudo apt install pipx | brew install pipx") if in_git_repo: hint() hint("`roar run` requires a clean git tree — runs are tagged with the commit SHA.") diff --git a/roar/cli/commands/register.py b/roar/cli/commands/register.py index 086fd6a1..5ce924c7 100644 --- a/roar/cli/commands/register.py +++ b/roar/cli/commands/register.py @@ -171,7 +171,7 @@ def _render_tag_summary(summary: RegisterTagSummary | None) -> None: def _apply_register_binds( ctx: RoarContext, *, - target: str, + target: str | None, response: RegisterLineageResponse, bind_targets: tuple[str, ...], no_bind: bool, @@ -190,7 +190,7 @@ def _apply_register_binds( rather than failing the command. """ refs: list[str] = [] - if not no_bind and response.artifact_hash: + if target is not None and not no_bind and response.artifact_hash: resolved_target = resolve_register_lineage_target( target, cwd=ctx.cwd, roar_dir=ctx.roar_dir ) @@ -487,15 +487,16 @@ def register( raise click.ClickException("--anonymous requires public visibility; remove --private.") target_was_defaulted = target is None + active_session_hash: str | None = None if target is None: - # No target -> register the whole active session. Resolving to the - # session's canonical hash routes through the session_hash collection - # path, which includes every job in the session (e.g. a downstream - # evaluate step), not just an artifact's upstream ancestry. + # Resolve a hash only for the confirmation preview. The application + # receives target=None so it selects the active session after publish + # bootstrap; bootstrap can change the creator identity and therefore + # the canonical hash. from ...application.query.status import StatusQueryError, compute_active_session_hash try: - target = compute_active_session_hash(ctx.roar_dir) + active_session_hash = compute_active_session_hash(ctx.roar_dir) except StatusQueryError as exc: raise click.ClickException(str(exc)) from exc @@ -510,7 +511,7 @@ def register( and not yes and not dry_run and not confirm_defaulted_active_session_publish( - session_hash=target, + session_hash=active_session_hash or "", command_name="roar register", start_dir=str(ctx.cwd), roar_dir=ctx.roar_dir, @@ -571,6 +572,7 @@ def register( web_url = _resolve_glaas_web_url(start_dir=str(ctx.cwd)) session_preview = _preview_hash(response.session_hash) if response.session_hash else "" session_url = _display_session_url(response.session_url, web_url, response.session_hash) + display_target = target if target is not None else "active session" # Apply the implicit binds up front so their result can be folded into the # register checklist (rather than printed as a separate trailing block). @@ -584,7 +586,7 @@ def register( # Format output if dry_run: - click.echo(f"Dry run: would register lineage for: {target}") + click.echo(f"Dry run: would register lineage for: {display_target}") click.echo(f" Session: {session_preview}") click.echo(f" Jobs: {response.jobs_registered}") click.echo(f" Artifacts: {response.artifacts_registered}") @@ -592,7 +594,7 @@ def register( # Secrets now ride as a line on the reproducibility punchlist below # ("no secrets in published lineage", note: none detected / N redacted). # Preview reproducibility BEFORE publishing (not yet on GLaaS). - _render_register_checklist(ctx, target, response, on_glaas=False, dry_run=True) + _render_register_checklist(ctx, display_target, response, on_glaas=False, dry_run=True) click.echo("") click.echo("GLaaS:") click.echo(f" Session: {session_url}") @@ -604,7 +606,7 @@ def register( for warning in response.warnings: click.echo(f"Warning: {warning}", err=True) _render_tag_summary(response.tag_summary) - click.echo(f"Already registered on GLaaS: {target}") + click.echo(f"Already registered on GLaaS: {display_target}") click.echo(f" Session: {session_preview}") click.echo(f" Labels: {response.labels_synced}") click.echo("") @@ -616,7 +618,7 @@ def register( else: for warning in response.warnings: click.echo(f"Warning: {warning}", err=True) - click.echo(f"Registered lineage for: {target}") + click.echo(f"Registered lineage for: {display_target}") click.echo(f" Session: {session_preview}") # Secrets ride as a punchlist line (see the checklist below). @@ -630,7 +632,7 @@ def register( # One punchlist: reproducibility checks + what register did (tag/push/ # counts + bind folded in), replacing the old separate stat/tag/bind blocks. _render_register_checklist( - ctx, target, response, on_glaas=True, bind_summaries=bind_summaries + ctx, display_target, response, on_glaas=True, bind_summaries=bind_summaries ) click.echo("") diff --git a/roar/cli/commands/reproduce.py b/roar/cli/commands/reproduce.py index be2da008..b6909ea8 100644 --- a/roar/cli/commands/reproduce.py +++ b/roar/cli/commands/reproduce.py @@ -59,6 +59,26 @@ default=None, help="Dump DAG lineage response to a JSON file", ) +@click.option( + "--export-requirements", + "export_requirements", + type=click.Path(), + default=None, + help="Write the recorded pip pins to a requirements.txt and exit (no run). " + "Debug a failed install with `pip install --dry-run -r ` to see which " + "pins don't resolve (yanked, private, or extra-index).", +) +@click.option( + "--step-timeout", + "step_timeout", + type=int, + default=None, + envvar="ROAR_REPRODUCE_STEP_TIMEOUT", + help="Per-step wall-clock timeout in seconds for --run. Default: no timeout " + "(a step may be slower on the reproducing host than on the one that made it). " + "Also settable via ROAR_REPRODUCE_STEP_TIMEOUT. On timeout the whole process " + "group is killed so no orphaned workload keeps burning compute.", +) @click.pass_obj def reproduce( ctx: RoarContext, @@ -73,6 +93,8 @@ def reproduce( package_sync: bool, list_requirements: bool, out_path: str | None, + export_requirements: str | None, + step_timeout: int | None, ) -> None: """Reproduce an artifact or lineage from a recorded hash. @@ -117,6 +139,8 @@ def reproduce( package_sync=package_sync, list_requirements=list_requirements, out_path=out_path, + export_requirements=export_requirements, + step_timeout=step_timeout, ) ) except ValueError as exc: diff --git a/roar/cli/publish_intent.py b/roar/cli/publish_intent.py index 55be7b3a..95282227 100644 --- a/roar/cli/publish_intent.py +++ b/roar/cli/publish_intent.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from dataclasses import dataclass from pathlib import Path @@ -18,6 +19,8 @@ class PublishIntent: def _is_logged_in() -> bool: """True iff there's a usable GLaaS/TReqs session on this machine.""" + if os.environ.get("ROAR_DELEGATED_AUTH") == "1": + return True try: from ..auth_store import load_auth_state @@ -46,6 +49,16 @@ def resolve_publish_intent( Deterministic, so it's headless-safe — no interactive prompt is required to reach a default. """ + if os.environ.get("ROAR_DELEGATED_AUTH") == "1": + # The workload is not the authority for attribution or visibility. The + # agent supplies the exact frozen project policy, which must beat repo + # config and command flags just as operational redirects beat saved + # config elsewhere. + return PublishIntent( + public=os.environ.get("ROAR_DELEGATED_VISIBILITY") == "public", + anonymous=False, + ) + if anonymous: return PublishIntent(public=True, anonymous=True) diff --git a/roar/core/interfaces/registration.py b/roar/core/interfaces/registration.py index e4cec771..b94a5f56 100644 --- a/roar/core/interfaces/registration.py +++ b/roar/core/interfaces/registration.py @@ -91,6 +91,9 @@ class BatchRegistrationResult: # (a full re-register): the existing DAG hash. The caller skips finalize and # reuses this hash instead of failing on duplicate jobs. already_registered_session_hash: str | None = None + # True only when GLaaS has server-bound the fresh registration session to + # that existing hash after exact-scope and complete-job-set verification. + existing_binding_prepared: bool = False @runtime_checkable diff --git a/roar/core/models/provenance.py b/roar/core/models/provenance.py index cd992605..57c1e5ee 100644 --- a/roar/core/models/provenance.py +++ b/roar/core/models/provenance.py @@ -67,6 +67,9 @@ class PythonInjectData(RoarBaseModel): installed_packages: dict[str, str] = Field(default_factory=dict) python_version: str = "" python_implementation: str = "" + # Health of the sitecustomize package-capture channel. A missing/invalid + # Python capture must not be confused with a successful empty package set. + capture_status: str = "missing" @computed_field # type: ignore[prop-decorator] @property diff --git a/roar/db/schema.py b/roar/db/schema.py index 7eefd3c0..b97eb05b 100644 --- a/roar/db/schema.py +++ b/roar/db/schema.py @@ -245,10 +245,29 @@ CREATE INDEX IF NOT EXISTS idx_hash_cache_path ON hash_cache(path); CREATE INDEX IF NOT EXISTS idx_hash_cache_updated ON hash_cache(cached_at); + +-- ============================================================================= +-- DELEGATED PUT OPERATIONS +-- Durable retry identity for one broker-backed put operation. A pending row is +-- reused after an interrupted response; completing it advances the ordinal for +-- the next command, even when the request is otherwise identical. +-- ============================================================================= +CREATE TABLE IF NOT EXISTS delegated_put_operations ( + task_identity TEXT NOT NULL, + session_id INTEGER NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + ordinal INTEGER NOT NULL, + request_fingerprint TEXT NOT NULL, + put_job_uid TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'completed')), + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + completed_at REAL, + PRIMARY KEY (task_identity, session_id) +); """ -_SCHEMA_VERSION = 3 # Bump when adding new migrations below. +_SCHEMA_VERSION = 5 # Bump when adding new migrations below. def run_migrations(conn) -> None: @@ -365,5 +384,39 @@ def run_migrations(conn) -> None: if "write_origin" not in label_columns: conn.execute("ALTER TABLE labels ADD COLUMN write_origin TEXT") + conn.execute( + """ + CREATE TABLE IF NOT EXISTS delegated_put_operations ( + task_identity TEXT NOT NULL, + session_id INTEGER NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + ordinal INTEGER NOT NULL, + request_fingerprint TEXT NOT NULL, + put_job_uid TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'completed')), + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + completed_at REAL, + PRIMARY KEY (task_identity, session_id) + ) + """ + ) + + delegated_put_columns = { + row["name"] + for row in conn.execute("PRAGMA table_info(delegated_put_operations)").fetchall() + } + if "put_job_uid" not in delegated_put_columns: + conn.execute( + "ALTER TABLE delegated_put_operations ADD COLUMN put_job_uid TEXT NOT NULL DEFAULT ''" + ) + conn.execute( + """ + UPDATE delegated_put_operations + SET put_job_uid = 'delegated-put-' || substr(task_identity, 1, 16) + || '-' || session_id || '-' || ordinal + WHERE put_job_uid = '' + """ + ) + # Stamp the schema version so subsequent opens skip the full migration check. conn.execute(f"PRAGMA user_version = {_SCHEMA_VERSION}") diff --git a/roar/execution/provenance/assembler.py b/roar/execution/provenance/assembler.py index 7600a44d..17c43829 100644 --- a/roar/execution/provenance/assembler.py +++ b/roar/execution/provenance/assembler.py @@ -103,6 +103,7 @@ def assemble(self, ctx: ProvenanceContext, config: dict[str, Any]) -> dict[str, }, "processes": ctx.process_summary, "runtime": self._runtime_to_dict(ctx.runtime_info), + "python_capture": ctx.python_data.capture_status, } # Add analyzer results diff --git a/roar/execution/provenance/data_loader.py b/roar/execution/provenance/data_loader.py index e57d4b02..b559e6a5 100644 --- a/roar/execution/provenance/data_loader.py +++ b/roar/execution/provenance/data_loader.py @@ -195,6 +195,7 @@ def load_python_data(self, path: str | None) -> PythonInjectData: return PythonInjectData( sys_prefix=sys.prefix, sys_base_prefix=sys.base_prefix, + capture_status="missing", ) try: @@ -206,6 +207,7 @@ def load_python_data(self, path: str | None) -> PythonInjectData: return PythonInjectData( sys_prefix=sys.prefix, sys_base_prefix=sys.base_prefix, + capture_status="invalid", ) return PythonInjectData( @@ -219,4 +221,5 @@ def load_python_data(self, path: str | None) -> PythonInjectData: installed_packages=data.get("installed_packages", {}), python_version=data.get("python_version", ""), python_implementation=data.get("python_implementation", ""), + capture_status="complete", ) diff --git a/roar/execution/provenance/service.py b/roar/execution/provenance/service.py index 80c1a391..9b8c1ea0 100644 --- a/roar/execution/provenance/service.py +++ b/roar/execution/provenance/service.py @@ -5,6 +5,7 @@ """ import os +import re import shutil from datetime import datetime, timezone from typing import Any @@ -125,6 +126,10 @@ def collect( len(tracer_data.processes), ) python_data = self._data_loader.load_python_data(python_log_path) + if python_data.capture_status == "missing" and not self._contains_python_process( + tracer_data.processes + ): + python_data.capture_status = "not-applicable" self.logger.debug( "Python data loaded: modules=%d, packages=%d", len(python_data.modules_files), @@ -260,6 +265,7 @@ def collect( "shared_libs": python_data.shared_libs, "used_packages": python_data.used_packages, "installed_packages": python_data.installed_packages, + "capture_status": python_data.capture_status, }, } analyzer_results = analyzers.run_analyzers(analyzer_context, config=config) @@ -286,6 +292,24 @@ def collect( self.logger.debug("Provenance collection complete") return result + @staticmethod + def _contains_python_process(processes: list[dict[str, Any]]) -> bool: + """Whether the native trace observed a Python interpreter process.""" + for process in processes: + command = process.get("command") or [] + if isinstance(command, str): + command = [command] + if not isinstance(command, list) or not command: + continue + # Wrapper processes such as `env PYTHONPATH=. python ...` may be + # the only process entry emitted by preload, so inspect every argv + # token for an interpreter executable rather than argv[0] alone. + for token in command: + executable = os.path.basename(str(token)).lower() + if re.fullmatch(r"python(?:\d+(?:\.\d+)*)?(?:\.exe)?", executable): + return True + return False + def _resolve_exec_program(self, command: list[str] | None) -> str | None: """Resolve the run's exec'd program (the user's argv[0]) to an abspath. diff --git a/roar/execution/recording/job_recording.py b/roar/execution/recording/job_recording.py index 914ac6c2..4d97de8b 100644 --- a/roar/execution/recording/job_recording.py +++ b/roar/execution/recording/job_recording.py @@ -392,6 +392,8 @@ def _build_metadata_json( metadata["packages"] = prov["executables"]["packages"] if prov.get("runtime"): metadata["runtime"] = prov["runtime"] + if prov.get("python_capture"): + metadata["python_capture"] = prov["python_capture"] if prov.get("analysis"): metadata["analysis"] = prov["analysis"] metadata["git"] = git_info diff --git a/roar/execution/reproduction/environment_setup.py b/roar/execution/reproduction/environment_setup.py index b47cef9d..79c6117a 100644 --- a/roar/execution/reproduction/environment_setup.py +++ b/roar/execution/reproduction/environment_setup.py @@ -149,7 +149,9 @@ def setup_in_place( # Create virtual environment, pinned to the recorded interpreter. self.logger.debug("Creating virtual environment...") - venv_dir = self._create_venv(repo_dir, self._recorded_python_version(pipeline)) + venv_dir = self._create_venv( + repo_dir, self._recorded_python_version(pipeline), auto_confirm=auto_confirm + ) self.logger.debug("Virtual environment created at: %s", venv_dir) # Initialize roar in the cloned repository @@ -212,6 +214,17 @@ def setup_in_place( if pip_warnings: for w in pip_warnings: self.logger.warning(w) + # Propagate a failed install instead of returning a healthy-looking + # EnvironmentInfo. Previously `success` was ignored, so an unresolved + # pin still produced "Environment ready" followed by a dead run. The + # reproduce service catches RuntimeError as "Environment setup failed". + if not success: + raise RuntimeError( + "Required pip packages from the recorded provenance could not be " + "installed — the reproduction environment is incomplete. Re-run with " + "--pip-any-version to install available versions, or " + "--export-requirements to inspect/try the exact pins yourself." + ) self.logger.debug("pip installation complete") self.logger.debug("Environment setup complete") @@ -458,14 +471,19 @@ def _clone_repository( return repo_dir - def _create_venv(self, repo_dir: Path, target_version: str | None = None) -> Path: + def _create_venv( + self, repo_dir: Path, target_version: str | None = None, auto_confirm: bool = False + ) -> Path: """ Create virtual environment in repository, pinned to the recorded Python. - We try the exact recorded interpreter (uv downloads a managed build if - needed), then the recorded major.minor, then fall back to the default - with a warning. We never block — a different interpreter still - reproduces, just less faithfully ("same setup" is best-effort). + With uv we provision the *exact* recorded interpreter (uv downloads a + managed build if needed). Without uv we can only use the interpreter roar + is running under; if that differs from the recorded one at the major.minor + level we warn, recommend uv, and — unless ``auto_confirm`` — ask before + continuing, because the recorded packages may not install or behave the + same (see :meth:`_confirm_python_mismatch`). We do not silently substitute + a different interpreter and then blame the package list. Returns: Path to venv directory @@ -479,7 +497,7 @@ def _create_venv(self, repo_dir: Path, target_version: str | None = None) -> Pat self._print("Creating virtual environment...") if self._use_uv: - self._create_venv_uv(venv_dir, repo_dir, target_version) + self._create_venv_uv(venv_dir, repo_dir, target_version, auto_confirm) else: # `python -m venv` can only use the running interpreter. subprocess.run( @@ -487,7 +505,7 @@ def _create_venv(self, repo_dir: Path, target_version: str | None = None) -> Pat check=True, cwd=repo_dir, ) - self._warn_python_mismatch(target_version, self._get_python_version()) + self._confirm_python_mismatch(target_version, self._get_python_version(), auto_confirm) gitignore = venv_dir / ".gitignore" if not gitignore.exists(): @@ -495,7 +513,13 @@ def _create_venv(self, repo_dir: Path, target_version: str | None = None) -> Pat return venv_dir - def _create_venv_uv(self, venv_dir: Path, repo_dir: Path, target_version: str | None) -> None: + def _create_venv_uv( + self, + venv_dir: Path, + repo_dir: Path, + target_version: str | None, + auto_confirm: bool = False, + ) -> None: """Create the venv with uv, pinned to the recorded interpreter if we can.""" for version in self._python_candidates(target_version): result = subprocess.run( @@ -513,7 +537,9 @@ def _create_venv_uv(self, venv_dir: Path, repo_dir: Path, target_version: str | # Couldn't provision the recorded interpreter — use uv's default, then warn. subprocess.run(["uv", "venv", str(venv_dir)], check=True, cwd=repo_dir) - self._warn_python_mismatch(target_version, self._venv_python_version(venv_dir)) + self._confirm_python_mismatch( + target_version, self._venv_python_version(venv_dir), auto_confirm + ) def _recorded_python_version(self, pipeline: "PipelineInfo") -> str | None: """The interpreter version recorded for this lineage (e.g. '3.14.4'), or None.""" @@ -535,21 +561,59 @@ def _python_candidates(target_version: str | None) -> list[str]: candidates.append(minor) return candidates - def _warn_python_mismatch(self, recorded: str | None, actual: str | None) -> None: - """Warn when the venv's interpreter differs from the recorded one at the - major.minor level. Patch differences (3.14.4 vs 3.14.6) aren't - reproducibility-relevant, so they don't warn.""" + def _confirm_python_mismatch( + self, recorded: str | None, actual: str | None, auto_confirm: bool + ) -> None: + """Handle a major.minor interpreter mismatch between capture and reproduce. + + Patch-level differences (3.14.4 vs 3.14.6) aren't reproducibility-relevant + and pass silently. For a major.minor mismatch the recorded packages (e.g. + ABI-tagged wheels) may fail to install or behave differently, so we warn + loudly, recommend uv (which provisions the *exact* recorded interpreter), + and — unless ``auto_confirm`` (``--yes``) — ask before continuing. + Declining aborts the reproduction rather than silently using the wrong + Python. + + Many pure-Python repos still reproduce fine on a different minor, which is + why this warns-and-asks rather than hard-failing. + """ if not recorded: return rec_minor = ".".join(recorded.split(".")[:2]) act_minor = ".".join((actual or "").split(".")[:2]) if act_minor and act_minor == rec_minor: return + using = actual or "a different interpreter" + uv_url = "https://docs.astral.sh/uv/getting-started/installation/" + bar = "=" * 64 self._print( - f"⚠ Recorded Python was {recorded}; reproducing with {using} — results may differ." + f"\n{bar}\n" + "⚠ PYTHON VERSION MISMATCH\n" + f" Recorded at capture: Python {recorded}\n" + f" Reproducing with: Python {using}\n" + f" The recorded packages were built for {rec_minor}; some (e.g.\n" + " ABI-tagged wheels) may fail to install or behave differently on " + f"{act_minor or 'this interpreter'}.\n\n" + " For a faithful reproduction, install uv and re-run — roar will then\n" + " provision the exact recorded interpreter automatically:\n" + f" {uv_url}\n" + f"{bar}" ) + if auto_confirm: + self._print("Continuing with the mismatched interpreter (--yes).") + return + + if not self._presenter.confirm( + f"Continue reproducing with Python {using} anyway?", default=False + ): + raise RuntimeError( + f"Reproduction aborted: recorded Python {recorded} is not available " + f"(no uv to provision it). Install uv ({uv_url}) or re-run with " + "--yes to proceed anyway." + ) + @staticmethod def _venv_python_version(venv_dir: Path) -> str | None: """Read the created venv's Python version from pyvenv.cfg, or None.""" diff --git a/roar/execution/reproduction/installers.py b/roar/execution/reproduction/installers.py index 37d4e978..38aeb30b 100644 --- a/roar/execution/reproduction/installers.py +++ b/roar/execution/reproduction/installers.py @@ -253,6 +253,8 @@ def install_packages( presenter: IPresenter | None = None, ) -> tuple[bool, list[str]]: warnings: list[str] = [] + unresolved_packages: list[str] = [] + conflict_in_combination = False active_presenter = presenter or self._presenter if not packages: self._print("No packages to install from provenance.") @@ -277,7 +279,18 @@ def install_packages( succeeded_packages.append(package) if succeeded_packages: - self._run_pip(venv_dir, repo_dir, ["install", *succeeded_packages], show_output=True) + combined = self._run_pip( + venv_dir, repo_dir, ["install", *succeeded_packages], show_output=True + ) + if combined.returncode != 0: + # Each of these pins resolved on its own (the per-package --dry-run + # above), but they conflict in combination: pip exited non-zero and + # left an incomplete/inconsistent venv. Individually-resolvable is + # NOT jointly-installable, so treat them as unresolved rather than + # discarding this return code and printing a false "Environment + # ready" over a venv missing most of its packages. + conflict_in_combination = True + unresolved_packages.extend(succeeded_packages) if failed_packages: self._print(f"\nExact versions not found for {len(failed_packages)} pip packages:") @@ -302,6 +315,7 @@ def install_packages( warnings.append( f"Some pip packages failed to install: {(fallback.stderr or '').strip()}" ) + unresolved_packages.extend(failed_packages) else: for package in failed_packages: warnings.append( @@ -310,6 +324,29 @@ def install_packages( else: for package in failed_packages: warnings.append(f"Skipped {package} (exact version not found)") + unresolved_packages.extend(failed_packages) + + if unresolved_packages: + # A recorded pin could not be installed. Do NOT report success — that + # produced a green "Pip package installation complete" / "Environment + # ready" banner followed by a dead run (ModuleNotFoundError). Fail + # honestly so the reproduction reports the env-setup failure instead. + self._print( + f"\nEnvironment is NOT reproducible: {len(unresolved_packages)} recorded " + "pip package(s) could not be installed:" + ) + for package in unresolved_packages: + self._print(f" - {package}") + if conflict_in_combination: + self._print( + "These pins resolve individually but conflict when installed " + "together, so pip left an incomplete environment." + ) + self._print( + "Re-run with --pip-any-version to install available versions instead, " + "or --export-requirements to inspect/try the exact pins yourself." + ) + return False, warnings self._print("Pip package installation complete") return True, warnings diff --git a/roar/execution/reproduction/pipeline_executor.py b/roar/execution/reproduction/pipeline_executor.py index 342b84f1..ad48a8fb 100644 --- a/roar/execution/reproduction/pipeline_executor.py +++ b/roar/execution/reproduction/pipeline_executor.py @@ -5,9 +5,11 @@ This service handles executing pipeline steps during reproduction. """ +import contextlib import json import os import shutil +import signal import subprocess import sys from typing import TYPE_CHECKING @@ -39,6 +41,7 @@ def __init__( self, presenter: "IPresenter | None" = None, roar_executable: str | None = None, + step_timeout: int | None = None, ): """ Initialize pipeline executor. @@ -46,10 +49,13 @@ def __init__( Args: presenter: Presenter for user feedback roar_executable: Path to roar executable (auto-detected if not provided) + step_timeout: Per-step wall-clock timeout in seconds; ``None`` (default) + means no timeout. """ self._presenter = presenter or NullPresenter() self._roar_initialized = False self._roar_executable = roar_executable or self._detect_roar_executable() + self._step_timeout = step_timeout def execute( self, @@ -153,31 +159,58 @@ def _run_step( # Set up environment env = self._prepare_environment(environment, env_vars=step_env_vars) - # Run the command + # Run the command in its own session/process group so that, if a timeout + # fires, we can kill the whole tree. shell=True means the direct child is + # a shell whose grandchild (e.g. train.py) would be orphaned by a plain + # kill of the shell — leaving a workload running on the GPU past the + # declared failure. `timeout` defaults to None (no timeout): a run should + # not be capped at an arbitrary wall-clock that also makes the row only + # reproducible on hardware at least as fast as the machine that made it. try: # Note: Using shell=True for complex commands with pipes, etc. - result = subprocess.run( + proc = subprocess.Popen( wrapped_command, shell=True, cwd=environment.repo_dir, env=env, - timeout=3600, # 1 hour timeout + start_new_session=True, ) + try: + returncode = proc.wait(timeout=self._step_timeout) + except subprocess.TimeoutExpired: + self._print( + f" Step timed out after {self._step_timeout}s — killing the process group" + ) + self._kill_process_group(proc) + return False - if result.returncode == 0: + if returncode == 0: self._print(" Success") return True else: - self._print(f" Failed with exit code {result.returncode}") + self._print(f" Failed with exit code {returncode}") return False - except subprocess.TimeoutExpired: - self._print(" Step timed out after 1 hour") - return False except Exception as e: self._print(f" Error: {e}") return False + @staticmethod + def _kill_process_group(proc: "subprocess.Popen[bytes]") -> None: + """SIGKILL the step's whole process group, then reap it. + + With ``shell=True`` the workload is a grandchild of the shell, so killing + only ``proc`` leaves it orphaned (a false failure + a silent GPU-cost + leak on someone else's bill). ``start_new_session=True`` gives the step + its own group, which we kill here. + """ + try: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + except (ProcessLookupError, PermissionError): + proc.kill() + with contextlib.suppress(subprocess.TimeoutExpired): + proc.wait(timeout=30) + def _wrap_with_roar( self, command: str, diff --git a/roar/execution/reproduction/pipeline_metadata.py b/roar/execution/reproduction/pipeline_metadata.py index 91f256d2..117783b8 100644 --- a/roar/execution/reproduction/pipeline_metadata.py +++ b/roar/execution/reproduction/pipeline_metadata.py @@ -68,6 +68,17 @@ def first_runtime(self, build_steps: list[dict], run_steps: list[dict]) -> dict[ return runtime return {} + def python_capture_complete(self, build_steps: list[dict], run_steps: list[dict]) -> bool: + """False when new lineage explicitly reports failed Python capture. + + Older lineage has no marker and remains backward-compatible. + """ + for step in [*build_steps, *run_steps]: + metadata = self._normalize_metadata(step.get("metadata")) + if metadata.get("python_capture") in {"missing", "invalid"}: + return False + return True + def summarize_requirements( self, build_steps: list[dict], run_steps: list[dict] ) -> RequirementSummary: diff --git a/roar/execution/runtime/coordinator.py b/roar/execution/runtime/coordinator.py index 848b2167..70ca873e 100644 --- a/roar/execution/runtime/coordinator.py +++ b/roar/execution/runtime/coordinator.py @@ -7,6 +7,8 @@ from __future__ import annotations +import contextlib +import glob import os import secrets import sys @@ -289,6 +291,13 @@ def stop_runtime_resources(exit_code: int | None) -> RuntimeObservationBundle: collect_dropped_paths=(ctx.verbosity == "debug"), command=list(ctx.command), ) + if prov.get("python_capture") in {"missing", "invalid"}: + self.presenter.print_error( + "warning: Python package capture did not complete; this job's package list is " + "incomplete and its runtime reproducibility check will fail.\n" + " Avoid replacing/removing PYTHONPATH, `env -i`, and Python -E/-I/-S. " + "Use `env -C python ...` when only a working-directory change is needed." + ) t_prov_end = time.perf_counter() n_read = len(prov.get("data", {}).get("read_files", [])) n_written = len(prov.get("data", {}).get("written_files", [])) @@ -472,3 +481,9 @@ def _cleanup_logs(self, tracer_log: str, inject_log: str) -> None: os.remove(log_file) except OSError: pass + # Sweep any per-PID inject-log shards that merge_inject_logs didn't reach + # (e.g. a report written after the merge, or a merge that never ran). + if inject_log: + for shard in glob.glob(glob.escape(inject_log) + ".*"): + with contextlib.suppress(OSError): + os.remove(shard) diff --git a/roar/execution/runtime/inject/sitecustomize.py b/roar/execution/runtime/inject/sitecustomize.py index 3e44a03f..3e25ca64 100644 --- a/roar/execution/runtime/inject/sitecustomize.py +++ b/roar/execution/runtime/inject/sitecustomize.py @@ -1,38 +1,179 @@ # ruff: noqa: E402 import atexit +import importlib.machinery import importlib.util import os import sys +_RUNTIME_CACHE_COLLISIONS_ENV = "ROAR_RUNTIME_CACHE_COLLISIONS" -def _prepend_roar_runtime_pythonpath() -> None: - """Prepend ``ROAR_RUNTIME_PYTHONPATH`` entries to ``sys.path`` (in order). - When the traced Python has a lazy-installed ABI-matched runtime tree on - ``ROAR_RUNTIME_PYTHONPATH``, that tree must beat system site-packages — - the system copies are the wrong-ABI ones, which is exactly why we - installed the tree in the first place. Prepending the whole list in - declared order (cache, then bundled fallbacks) keeps the lazy-install - cache at ``sys.path[0]``. +def _roar_runtime_cache_root() -> str: + """``$XDG_CACHE_HOME/roar/runtime`` (default ``~/.cache/roar/runtime``). - Logic is inlined (rather than imported from elsewhere in roar) because - this runs *before* roar is necessarily importable — making roar - importable is exactly what this function does. + Inlined to match ``lazy_install.runtime_cache_root()`` — this runs before + roar is importable, so it can't call into roar. + """ + xdg = os.environ.get("XDG_CACHE_HOME") + base = xdg if xdg else os.path.join(os.path.expanduser("~"), ".cache") + return os.path.abspath(os.path.join(base, "roar", "runtime")) + + +def _path_key(path: str) -> str: + """Normalize a path for comparisons without changing import ordering.""" + return os.path.normcase(os.path.realpath(os.path.abspath(path or os.curdir))) + + +def _runtime_pythonpath_entries() -> list[str]: + return [ + path for path in os.environ.get("ROAR_RUNTIME_PYTHONPATH", "").split(os.pathsep) if path + ] + + +def _workload_search_path() -> list[str]: + """Return the original workload import roots, excluding Roar-owned paths.""" + roar_paths = {_path_key(path) for path in _runtime_pythonpath_entries()} + roar_paths.update( + _path_key(path) + for path in os.environ.get("ROAR_RUNTIME_PYTHONPATH_ACTIVE", "").split(os.pathsep) + if path + ) + roar_paths.add(_path_key(os.path.dirname(os.path.abspath(__file__)))) + return [path for path in sys.path if _path_key(path) not in roar_paths] + + +def _top_level_import_names(paths: list[str]) -> set[str]: + """Discover import names supplied by one or more site-packages trees.""" + names: set[str] = set() + import_suffixes = sorted( + { + *importlib.machinery.SOURCE_SUFFIXES, + *importlib.machinery.BYTECODE_SUFFIXES, + *importlib.machinery.EXTENSION_SUFFIXES, + }, + key=len, + reverse=True, + ) + for path in paths: + try: + entries = os.scandir(path) + except OSError: + continue + with entries: + for entry in entries: + entry_name = entry.name + if entry_name.startswith(".") or entry_name == "__pycache__": + continue + if entry_name.endswith((".dist-info", ".egg-info", ".data")): + continue + try: + if entry.is_dir(): + candidate = entry_name + elif entry.is_file(): + candidate = "" + for suffix in import_suffixes: + if entry_name.endswith(suffix): + candidate = entry_name[: -len(suffix)] + break + else: + continue + except OSError: + continue + if candidate.isidentifier(): + names.add(candidate) + return names + + +def _runtime_cache_collisions(cache_paths: list[str]) -> tuple[str, ...]: + """Names a cache would shadow on the workload's unmodified search path.""" + workload_paths = _workload_search_path() + collisions: list[str] = [] + for name in sorted(_top_level_import_names(cache_paths)): + try: + spec = importlib.machinery.PathFinder.find_spec(name, workload_paths) + except Exception: + # Detection uncertainty must degrade rather than risk changing the workload. + collisions.append(name) + continue + if spec is not None: + collisions.append(name) + return tuple(collisions) + + +def _record_runtime_cache_collisions(collisions: tuple[str, ...]) -> None: + if collisions: + os.environ[_RUNTIME_CACHE_COLLISIONS_ENV] = ",".join(collisions) + else: + os.environ.pop(_RUNTIME_CACHE_COLLISIONS_ENV, None) + + +def _set_active_runtime_paths(paths: list[str]) -> None: + if paths: + os.environ["ROAR_RUNTIME_PYTHONPATH_ACTIVE"] = os.pathsep.join(paths) + else: + os.environ.pop("ROAR_RUNTIME_PYTHONPATH_ACTIVE", None) + + +def _add_active_runtime_path(path: str, *, prepend: bool = False) -> None: + active = [ + entry + for entry in os.environ.get("ROAR_RUNTIME_PYTHONPATH_ACTIVE", "").split(os.pathsep) + if entry + ] + if path in active: + return + if prepend: + active.insert(0, path) + else: + active.append(path) + _set_active_runtime_paths(active) + + +def _add_roar_runtime_pythonpath() -> None: + """Make roar importable in the traced process **without letting roar's own + environment shadow the workload's recorded packages**. + + ``ROAR_RUNTIME_PYTHONPATH`` carries two very different kinds of entry: + + - roar's lazy-installed **ABI-matched runtime cache** + (``~/.cache/roar/runtime//site-packages``). This *must* beat the + system's wrong-ABI copies — that is why it was installed — so it is + **prepended only when none of its import names overlap the workload**. + - roar's package root / the parent interpreter's **site-packages**, added so + a non-editable or cross-interpreter child can import roar at all. These are + **appended**, so the workload's own venv always wins. + + Prepending the second kind was **P0-14**: when roar ran under a different + interpreter than the child (e.g. roar under system 3.10, workload venv 3.12), + its host ``dist-packages`` landed at ``sys.path[0]`` and shadowed the recorded + pins — the run executed against host packages and could certify GREEN for the + wrong reason. roar's core injection is pure-Python, so appending still leaves + it importable; ABI-specific backend deps are handled separately by the runtime + gate below. + + Inlined (not imported from roar) because this runs before roar is importable. """ if importlib.util.find_spec("roar") is not None: return - new_paths = [ - path - for path in os.environ.get("ROAR_RUNTIME_PYTHONPATH", "").split(os.pathsep) - if path and path not in sys.path - ] + new_paths = [path for path in _runtime_pythonpath_entries() if path not in sys.path] if not new_paths: return - sys.path[:0] = new_paths - os.environ["ROAR_RUNTIME_PYTHONPATH_ACTIVE"] = os.pathsep.join(new_paths) + cache_root = _roar_runtime_cache_root() + must_win = [p for p in new_paths if os.path.abspath(p).startswith(cache_root + os.sep)] + others = [p for p in new_paths if p not in must_win] + collisions = _runtime_cache_collisions(must_win) if must_win else () + _record_runtime_cache_collisions(collisions) + active_paths: list[str] = [] + if must_win and not collisions: + sys.path[:0] = must_win # ABI-matched cache must beat wrong-ABI system copies + active_paths.extend(must_win) + if others: + sys.path.extend(others) # roar's env must NOT shadow the workload's venv (P0-14) + active_paths.extend(others) + _set_active_runtime_paths(active_paths) -_prepend_roar_runtime_pythonpath() +_add_roar_runtime_pythonpath() from roar.execution.framework.runtime_imports import RuntimeImportController from roar.execution.runtime.inject.support import ( @@ -94,14 +235,26 @@ def _repair_runtime_in_process(expected_soabi: str) -> bool: if tree is None: return False tree_str = str(tree) + collisions = _runtime_cache_collisions([tree_str]) + _record_runtime_cache_collisions(collisions) + if collisions: + return False if tree_str not in sys.path: sys.path.insert(0, tree_str) + _add_active_runtime_path(tree_str, prepend=True) return matching_compiled_pydantic_core(sys.path, expected_soabi) def _runtime_gate_degrade_message(running_abi: tuple[int, int]) -> str: + collisions = os.environ.get(_RUNTIME_CACHE_COLLISIONS_ENV, "") + collision_message = ( + f" Runtime cache disabled to preserve workload imports: {collisions}.\n" + if collisions + else "" + ) return ( f"roar: no ABI-matched runtime found for Python {running_abi[0]}.{running_abi[1]}.\n" + f"{collision_message}" f" Backend integrations (Ray, OSMO) are disabled for this run.\n" f" File I/O is still captured.\n" f" Fix one of:\n" diff --git a/roar/execution/runtime/inject/tracker.py b/roar/execution/runtime/inject/tracker.py index 24d067df..e2b70019 100644 --- a/roar/execution/runtime/inject/tracker.py +++ b/roar/execution/runtime/inject/tracker.py @@ -4,6 +4,7 @@ import builtins import contextlib +import glob import json import os import platform @@ -48,27 +49,80 @@ def get_loaded_shared_libs(real_open) -> list[str]: return sorted(libs) -def get_installed_packages() -> dict[str, str]: +def get_installed_packages( + excluded_paths: Sequence[str] = (), +) -> dict[str, str]: packages: dict[str, str] = {} try: from importlib import metadata as importlib_metadata for dist in importlib_metadata.distributions(): + try: + distribution_root = str(dist.locate_file("")) + except Exception: + distribution_root = "" + if distribution_root and is_under_any_runtime_path(distribution_root, excluded_paths): + continue metadata = cast(Mapping[str, str], dist.metadata) name = metadata.get("Name", None) version = metadata.get("Version", None) if name and version: - packages[name] = version + # Import resolution and distributions() both follow sys.path order. + # Preserve the first visible workload distribution for duplicate names. + packages.setdefault(name, version) except Exception: pass return packages +def _dist_is_in_repo(dist_name: str, repo_root: str) -> bool: + """True if ``dist_name``'s metadata resolves inside ``repo_root`` — i.e. it is + the workload's OWN package (an editable ``pip install -e .``, or the leftover + ``.egg-info`` a later ``pip uninstall`` doesn't remove), not a real + third-party dependency installed under site-packages.""" + try: + from importlib import metadata as importlib_metadata + + dist = importlib_metadata.distribution(dist_name) + path = getattr(dist, "_path", None) or dist.locate_file("") + return os.path.abspath(str(path)).startswith(repo_root + os.sep) + except Exception: + return False + + +# Package install roots. "dist-packages" (Debian/Ubuntu system Python, e.g. the +# cert AMIs) must be recognized alongside "site-packages" — otherwise packages +# installed there are dropped from the freeze entirely (P0-18): the workload +# imports them, but the file pass ignored them and the aliased-only name pass +# (P0-13 fix) doesn't rescue a normally-loaded import. +_PACKAGE_ROOT_MARKERS = ("site-packages/", "dist-packages/") + + +def _site_packages_top(fpath: str) -> str | None: + """The top-level package dir for a file under a package install root, else None.""" + for marker in _PACKAGE_ROOT_MARKERS: + idx = fpath.find(marker) + if idx < 0: + continue + top = fpath[idx + len(marker) :].split("/")[0] + if top.endswith(".py"): + top = top[:-3] + if top.startswith("_") or top.endswith((".dist-info", ".egg-info", ".so")): + return None + return top + return None + + def get_used_packages( modules_files: Sequence[str], installed_packages: Mapping[str, str | None], + imported_modules: Sequence[str] = (), + workload_root: str | None = None, + loaded_files: Mapping[str, str] | None = None, ) -> dict[str, str | None]: used: dict[str, str | None] = {} + repo_root = os.path.abspath(workload_root) if workload_root else None + loaded = loaded_files or {} try: from importlib import metadata as importlib_metadata @@ -79,18 +133,14 @@ def get_used_packages( try: for fpath in modules_files: - if "site-packages" not in fpath: + top_dir = _site_packages_top(fpath) + if top_dir is None: continue - idx = fpath.find("site-packages/") - if idx < 0: - continue - after_sp = fpath[idx + len("site-packages/") :] - top_dir = after_sp.split("/")[0] - if top_dir.endswith(".py"): - top_dir = top_dir[:-3] - if top_dir.endswith(".dist-info") or top_dir.endswith(".egg-info"): - continue - if top_dir.startswith("_") or top_dir.endswith(".so"): + if top_dir == "roar": + # roar records itself otherwise. roar-cli is installed separately + # and unpinned by _install_roar, so the pin is always redundant — + # harmless noise on a PyPI release, but fatal on an unpublished + # build (roar-cli==X.Y.dev0 can't resolve). P0-11. continue pkg_names = pkg_dist_map.get(top_dir, []) @@ -103,9 +153,103 @@ def get_used_packages( except Exception: pass + # Recover packages the workload IMPORTED but that the file pass mis-attributed + # because the import was ALIASED — e.g. a `sys.modules["wandb"] = trackio` + # logging shim leaves the loaded module's __file__ pointing at trackio, so the + # file pass records trackio and never wandb, yet the job genuinely needs wandb + # (its dist metadata is queried; its install is required). + # + # Scope this strictly to the aliased case: attribute a name only when it was + # imported AND the module actually loaded for it lives in a DIFFERENT + # site-packages package than the name. This is precisely what the file pass + # cannot see. It deliberately excludes: + # - normally-loaded imports (name == loaded package) -> the file pass's job; + # - merely-probed optional imports that happen to be installed (e.g. + # accelerate probing `sagemaker` on a SageMaker AMI) -> not loaded as an + # alias, so not attributed. Attributing those poisoned the freeze with + # unsatisfiable substrate pins — P0-13 (#264 regression). + # A never-imported package can never appear; the tracer's own package and the + # workload's own (editable/self) package are excluded as well. + try: + for name in imported_modules: + top = name.split(".")[0] + if not top or top.startswith("_") or top == "roar": + continue + loaded_file = loaded.get(top) + if not loaded_file: + continue # not actually loaded (find_spec probe / lazy import) + loaded_top = _site_packages_top(loaded_file) + if loaded_top is None or loaded_top == top: + continue # loaded as itself / not under site-packages -> file pass handles it + for pkg_name in pkg_dist_map.get(top, []): + if pkg_name not in installed_packages or pkg_name in used: + continue + # The workload's OWN package (editable / leftover .egg-info in the + # repo) is not a third-party dep — P0-12. + if repo_root and _dist_is_in_repo(pkg_name, repo_root): + continue + used[pkg_name] = installed_packages[pkg_name] + except Exception: + pass + return used +_MERGE_LIST_FIELDS = ("opened_files", "imported_modules", "modules_files", "shared_libs") +_MERGE_DICT_FIELDS = ("used_packages", "installed_packages", "env_reads") + + +def merge_inject_logs(base_path: str) -> None: + """Union per-PID inject-log shards (``{base_path}.``) into one record at + ``base_path``. + + Every process in a traced tree writes its own shard (see + :meth:`RuntimeInjectionTracker.write_log`). Unioning them recovers the full + workload — packages, files and imports seen by the parent AND by any worker — + instead of whichever process happened to write last. Set/dict activity is + unioned; scalar identity (``argv``, ``python_version``, ...) is taken from the + richest shard, i.e. the one that imported the most modules: multiprocessing + workers (``python -c ...``) import a subset, the workload imports everything. + + A no-op if there are no shards (e.g. the tracer produced no report). + """ + shards: list[tuple[str, dict]] = [] + for path in sorted(glob.glob(glob.escape(base_path) + ".*")): + try: + with open(path) as handle: + shards.append((path, json.load(handle))) + except (OSError, ValueError): + continue + if not shards: + return + + # Richest shard = most imported modules -> the workload, not a worker. + primary = max((data for _, data in shards), key=lambda d: len(d.get("modules_files") or [])) + merged: dict[str, Any] = dict(primary) + + for field in _MERGE_LIST_FIELDS: + union: set[str] = set() + for _, data in shards: + union.update(data.get(field) or []) + merged[field] = sorted(union) + + for field in _MERGE_DICT_FIELDS: + combined: dict[str, Any] = {} + for _, data in shards: + for key, value in (data.get(field) or {}).items(): + # Prefer a concrete version over a None placeholder. + if key not in combined or combined[key] is None: + combined[key] = value + merged[field] = dict(sorted(combined.items())) + + with open(base_path, "w") as handle: + json.dump(merged, handle) + + for path, _ in shards: + with contextlib.suppress(OSError): + os.remove(path) + + def get_active_runtime_pythonpath(environ: Mapping[str, str]) -> tuple[str, ...]: entries: list[str] = [] for raw_path in environ.get("ROAR_RUNTIME_PYTHONPATH_ACTIVE", "").split(os.pathsep): @@ -119,8 +263,52 @@ def get_active_runtime_pythonpath(environ: Mapping[str, str]) -> tuple[str, ...] def is_under_any_runtime_path(path: str, runtime_paths: Sequence[str]) -> bool: if not runtime_paths: return False - abs_path = os.path.abspath(path) - return any(abs_path.startswith(runtime_path) for runtime_path in runtime_paths) + abs_path = os.path.normcase(os.path.abspath(path)) + for runtime_path in runtime_paths: + abs_runtime_path = os.path.normcase(os.path.abspath(runtime_path)) + try: + if os.path.commonpath([abs_path, abs_runtime_path]) == abs_runtime_path: + return True + except ValueError: + continue + return False + + +def _roar_site_packages_root(inject_dir: str) -> str | None: + """roar's own install root — the site-packages (or ``uv tool`` venv) holding + the roar package. ``inject_dir`` is ``/roar/execution/runtime/inject``, + so the root is four levels up. None if the shape is unexpected.""" + root = os.path.abspath(inject_dir) + for _ in range(4): + parent = os.path.dirname(root) + if parent == root: + return None + root = parent + return root + + +def roar_footprint_paths(inject_dir: str, sys_prefix: str) -> tuple[str, ...]: + """Location(s) whose loaded modules are roar's OWN footprint, to subtract from + the freeze by PATH — never by name. + + The campaign runs roar in its own ``uv tool`` venv, ABI-matched to the workload + (the mandatory P0-14 layout). There ``ROAR_RUNTIME_PYTHONPATH_ACTIVE`` is null, + so the runtime-path filter is inert and roar's dependency footprint leaks into + the freeze — P0-11 (broad). roar's install root is knowable structurally, so we + exclude modules loaded from it. This distinguishes roar's copy of a package + from a same-named copy in the workload's venv (a different path); name-keying + could not, and stripped the workload's own tqdm / typing-extensions — P0-28. + + Guarded to the ISOLATED case: when roar shares the workload venv (its root is + under the interpreter's own prefix), path cannot tell the copies apart, so this + returns nothing and the freeze safely OVER-includes rather than risk dropping a + workload dependency.""" + root = _roar_site_packages_root(inject_dir) + if not root: + return () + if is_under_any_runtime_path(root, (sys_prefix,)): + return () # shared venv — do not path-exclude (over-include is the safe side) + return (root,) class RuntimeInjectionTracker: @@ -153,10 +341,69 @@ def __init__( def install(self) -> None: """Patch builtins and environ access for activity capture.""" + self._install_fork_worker_finalizer() builtins.open = self.tracking_open builtins.__import__ = self.tracking_import setattr(self._environ, _ENVIRON_GET_METHOD_NAME, self.patched_environ_get) + def _install_fork_worker_finalizer(self) -> None: + """Make multiprocessing fork workers emit their per-PID inject shard. + + ``multiprocessing`` workers terminate through ``os._exit``, bypassing + Python's ordinary atexit handlers. Its own shutdown path does run + child-local ``Finalize`` callbacks, so register one after each fork. The + existing parent-side shard merger then sees the worker report exactly as + intended by PR #265. + + Workers are not always asked to stop, though. ``Pool.__exit__`` is + ``terminate()``, which SIGTERMs every worker, and ``util._exit_function`` + does the same to surviving daemon children -- the ``DataLoader`` shape. + A killed worker runs no exit hook at all, so the child also writes its + shard *immediately* after forking; see ``_register_in_fork_child``. + + Deliberately NOT done here: installing a SIGTERM handler. A Python + signal handler only runs when the interpreter reaches a bytecode + boundary, so a worker inside a long C call (BLAS, zlib, pickle) would + latch the signal and never die -- and both ``Pool._terminate_pool`` and + ``util._exit_function`` join workers with no timeout. Measured: a worker + in ``zlib.compress`` went from exit -15 in 0.02s to still alive after + 8s. Hanging the workload is far worse than a thin shard. + """ + try: + from multiprocessing import util as multiprocessing_util + + multiprocessing_util.register_after_fork( + self, RuntimeInjectionTracker._register_in_fork_child + ) + except Exception: + pass + + @staticmethod + def _register_in_fork_child(tracker: RuntimeInjectionTracker) -> None: + """Give the child a shard now, and a complete one if it exits orderly. + + The eager write is what survives a worker that is killed rather than + joined: it holds the state inherited at fork, which is the parent's + whole import set. The finalizer then rewrites the same per-PID path on + an orderly exit, upgrading it with whatever the worker imported while it + ran. Both paths are union-merged by ``merge_inject_logs``, so the + upgrade is free and a killed worker still contributes. + + The remaining boundary, stated plainly: imports a worker makes *after* + forking are lost if it is killed before exiting. Closing that needs + incremental journaling, not an exit hook. + """ + with contextlib.suppress(Exception): + tracker.write_log() + with contextlib.suppress(Exception): + from multiprocessing import util as multiprocessing_util + + multiprocessing_util.Finalize( + None, + tracker.write_log, + exitpriority=-100, + ) + def tracking_open(self, *args, **kwargs): if is_suppressed(): return self._real_open(*args, **kwargs) @@ -187,6 +434,12 @@ def write_log(self) -> None: return runtime_pythonpath = get_active_runtime_pythonpath(self._environ) + # Also exclude roar's own install root by LOCATION. In the campaign's + # ABI-matched uv-tool layout ROAR_RUNTIME_PYTHONPATH_ACTIVE is null, so the + # runtime-path filter alone leaves roar's dependency footprint in the freeze + # (P0-11 broad). Path-keyed, never name-keyed (P0-28), so a same-named + # workload copy in a different venv survives. + exclusion_paths = runtime_pythonpath + roar_footprint_paths(self._inject_dir, sys.prefix) modules_files = sorted( os.path.abspath(getattr(module, "__file__", "")) for module in sys.modules.values() @@ -194,11 +447,28 @@ def write_log(self) -> None: and not os.path.abspath(getattr(module, "__file__", "")).startswith(self._inject_dir) and not is_under_any_runtime_path( os.path.abspath(getattr(module, "__file__", "")), - runtime_pythonpath, + exclusion_paths, ) ) - installed_packages = get_installed_packages() - used_packages = get_used_packages(modules_files, installed_packages) + # Trev's #268 + P0-11 broad: exclude roar's runtime-tree AND install-root + # dists from the installed set, so the file pass can't resolve them. + installed_packages = get_installed_packages(excluded_paths=exclusion_paths) + # name -> loaded module file, so get_used_packages can tell an ALIASED + # import (sys.modules[name] resolves to a different package) from a normal + # or merely-probed one. Keyed by the sys.modules key (the import name), + # whose __file__ may point at the alias target. + loaded_files = { + name: os.path.abspath(getattr(module, "__file__", "")) + for name, module in sys.modules.items() + if getattr(module, "__file__", None) + } + used_packages = get_used_packages( + modules_files, + installed_packages, + sorted(self.imported_modules), + workload_root=os.getcwd(), + loaded_files=loaded_files, + ) data = { "opened_files": sorted(self.opened_files), "imported_modules": sorted(self.imported_modules), @@ -214,8 +484,18 @@ def write_log(self) -> None: "used_packages": used_packages, "python_version": platform.python_version(), "python_implementation": platform.python_implementation(), + "pid": os.getpid(), + "ppid": os.getppid(), } - with self._real_open(self._log_file, "w") as handle: + # Write to a PER-PID shard, not the shared ROAR_LOG_FILE. Every process in + # a traced tree (litdata/DataLoader workers, HF datasets num_proc, torchrun + # ranks, any multiprocessing spawn) inherits the same ROAR_LOG_FILE and + # runs this at exit; opening it "w" means each truncates the others, so the + # surviving record was whichever process wrote LAST — often a worker with a + # subset of the imports (or none of them), not the workload. Sharding by + # pid lets merge_inject_logs() union the full tree afterwards. + shard_path = f"{self._log_file}.{os.getpid()}" + with self._real_open(shard_path, "w") as handle: json.dump(data, handle) diff --git a/roar/execution/runtime/lazy_install.py b/roar/execution/runtime/lazy_install.py index b5afc2f0..66a5ee05 100644 --- a/roar/execution/runtime/lazy_install.py +++ b/roar/execution/runtime/lazy_install.py @@ -6,9 +6,9 @@ module installs a matching tree of runtime deps on demand into a per-ABI cache directory under ``~/.cache/roar/runtime//``. -``sitecustomize.py``'s ``_append_roar_runtime_pythonpath`` prepends the -cache directory to ``sys.path`` in the traced process, so imports there -resolve to the ABI-matched copies before reaching roar's bundled tree. +``sitecustomize.py`` prepends the cache directory only when its import names +do not collide with the workload. A collision degrades optional backend +dispatch instead of changing which packages the workload imports. """ from __future__ import annotations diff --git a/roar/execution/runtime/tracer.py b/roar/execution/runtime/tracer.py index 462c1615..939082fc 100644 --- a/roar/execution/runtime/tracer.py +++ b/roar/execution/runtime/tracer.py @@ -21,6 +21,7 @@ from ...core.models.run import TracerResult from ...core.tracer_modes import TRACER_BACKEND_ORDER, is_valid_tracer_mode from ...execution.runtime import tracer_backends +from ...execution.runtime.inject.tracker import merge_inject_logs class TracerService: @@ -114,8 +115,9 @@ def _lazy_install_runtime_entries( ) -> list[str]: """Probe the target Python and lazy-install a matching runtime tree on mismatch. - Returns a list of site-packages paths to prepend to - ``ROAR_RUNTIME_PYTHONPATH``. Empty on: + Returns a list of ABI-matched site-packages paths for + ``ROAR_RUNTIME_PYTHONPATH``. ``sitecustomize`` activates a returned + path only when it cannot shadow a workload import. Empty on: - non-Python targets (bash, make, etc.) — can't probe a python ABI; - matching ABI — bundled deps work as-is; - ``runtime.install = skip`` — opted out; @@ -619,6 +621,13 @@ def execute( signal_handler.restore() self.logger.debug("Signal handler restored") + # Union the per-PID inject-log shards written by every process in the tree + # into the single canonical inject_log_file the collector reads. Without + # this, a multiprocessing worker's shard would be the only record (each + # process used to truncate a shared log); merging recovers the workload's + # full package/file/import set. + merge_inject_logs(inject_log_file) + end_time = time.time() duration = end_time - start_time self.logger.debug( diff --git a/roar/filters/omit.py b/roar/filters/omit.py index 7c3b02ce..ebd85356 100644 --- a/roar/filters/omit.py +++ b/roar/filters/omit.py @@ -77,10 +77,10 @@ def was_modified(self) -> bool: re.compile(r"(sk-ant-[a-zA-Z0-9\-]+)"), "[ANTHROPIC_KEY_REDACTED]", ), - # HuggingFace token + # HuggingFace token — {20,} (not exact-34) to catch token-length variants (P0-16) ( "huggingface_token", - re.compile(r"(hf_[a-zA-Z0-9]{34})"), + re.compile(r"(hf_[A-Za-z0-9]{20,})"), "[HF_TOKEN_REDACTED]", ), # GitLab personal/project/CI access tokens diff --git a/roar/integrations/config/access.py b/roar/integrations/config/access.py index db67cb5f..ee1475a0 100644 --- a/roar/integrations/config/access.py +++ b/roar/integrations/config/access.py @@ -131,6 +131,8 @@ "GITHUB_TOKEN", "DATABASE_URL", "AWS_SECRET_ACCESS_KEY", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", ], "description": "Env var names whose values should be redacted (comma-separated)", }, diff --git a/roar/integrations/config/raw.py b/roar/integrations/config/raw.py index f4c85d48..ab3a968b 100644 --- a/roar/integrations/config/raw.py +++ b/roar/integrations/config/raw.py @@ -28,6 +28,8 @@ "GITHUB_TOKEN", "DATABASE_URL", "AWS_SECRET_ACCESS_KEY", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", ] }, "patterns": [], diff --git a/roar/integrations/config/schema.py b/roar/integrations/config/schema.py index ab29c617..f853b93f 100644 --- a/roar/integrations/config/schema.py +++ b/roar/integrations/config/schema.py @@ -109,6 +109,8 @@ class EnvVarsConfig(ConfigBaseModel): "GITHUB_TOKEN", "DATABASE_URL", "AWS_SECRET_ACCESS_KEY", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", ] ) @@ -133,6 +135,8 @@ class OmitConfig(ConfigBaseModel): enabled: bool = True secrets: SecretsConfig = Field(default_factory=SecretsConfig) env_vars: EnvVarsConfig = Field(default_factory=EnvVarsConfig) + # Value regexes (incl. the HF token) live in filters.omit.BUILTIN_PATTERNS, + # which is applied unconditionally and can't be disabled by config. patterns: list[CustomPattern] = Field(default_factory=list) allowlist: AllowlistConfig = Field(default_factory=AllowlistConfig) diff --git a/roar/integrations/glaas/client.py b/roar/integrations/glaas/client.py index 470167b1..804a7c30 100644 --- a/roar/integrations/glaas/client.py +++ b/roar/integrations/glaas/client.py @@ -202,6 +202,8 @@ def _request( def _make_auth_header(self, method: str, path: str, body: bytes | None = None) -> str | None: if self._force_anonymous: return None + if self._publish_auth.delegated_auth_available: + return None if self._publish_auth.access_token and not self._bearer_auth_rejected: return f"Bearer {self._publish_auth.access_token}" return make_auth_header(method, path, body) @@ -214,7 +216,11 @@ def _make_ssh_auth_header( return make_auth_header(method, path, body) def _can_fallback_from_bearer_to_ssh(self) -> bool: - if self._force_anonymous or self._bearer_auth_rejected: + if ( + self._force_anonymous + or self._bearer_auth_rejected + or self._publish_auth.delegated_auth_available + ): return False if not self._publish_auth.access_token: return False @@ -231,6 +237,8 @@ def probe_publish_auth(self) -> bool | None: """ if self._force_anonymous: return False + if self._publish_auth.delegated_auth_available: + return True if self._publish_auth.access_token: return True if not self.base_url: @@ -373,6 +381,21 @@ def register_composite_artifact( result, error = self._request("POST", "/api/v1/artifacts/composites", payload) return result, error + def register_composite_artifact_under_registration_session( + self, + registration_session_id: str, + payload: dict[str, Any], + ) -> tuple[dict | None, str | None]: + """Stage immutable composite metadata before session finalization.""" + body = {key: value for key, value in payload.items() if key != "session_hash"} + return self._request( + "POST", + f"/api/v1/registration-sessions/{registration_session_id}/artifacts/composites", + body, + auth_header_value=self._registration_session_auth_header(), + allow_auth_fallback=False, + ) + def get_composite_components(self, hash_prefix: str) -> tuple[dict | None, str | None]: """ Fetch stored component membership rows for a composite artifact. @@ -680,8 +703,6 @@ def finalize_registration_session( allow_auth_fallback=False, ) error = _normalize_scope_error(self._publish_auth.scope_request, error) - if error is None and self._registration_session_mode == "anonymous_public": - self._clear_registration_session_auth() return result, error def sync_labels( @@ -693,6 +714,25 @@ def sync_labels( return {"created": 0, "updated": 0, "unchanged": 0}, None return self._request("POST", "/api/v1/labels/sync", {"labels": labels}) + def sync_labels_under_registration_session( + self, + registration_session_id: str, + labels: list[dict[str, Any]], + ) -> tuple[dict | None, str | None]: + """Sync labels only to lineage finalized by this registration session.""" + if not labels: + return {"created": 0, "updated": 0, "noops": 0}, None + result, error = self._request( + "POST", + f"/api/v1/registration-sessions/{registration_session_id}/labels/batch", + {"labels": labels}, + auth_header_value=self._registration_session_auth_header(), + allow_auth_fallback=False, + ) + if error is None and self._registration_session_mode == "anonymous_public": + self._clear_registration_session_auth() + return result, error + def reconcile_labels( self, payload: dict[str, Any], @@ -810,6 +850,7 @@ def register_jobs_batch_under_registration_session( "already_registered": [ str(h) for h in (result.get("already_registered_session_hashes") or []) if h ], + "existing_binding_prepared": bool(result.get("existing_binding_prepared", False)), } return result.get("job_ids", []), result.get("errors", []), None, counts @@ -852,6 +893,21 @@ def register_job_view_edges( body: dict[str, Any] = {"view_edges": view_edges} return self._request("POST", f"/api/v1/jobs/{job_uid}/artifacts", body) + def register_job_view_edges_under_registration_session( + self, + registration_session_id: str, + job_uid: str, + view_edges: list[dict], + ) -> tuple[dict | None, str | None]: + """Stage view edges while both the job and composite are private.""" + return self._request( + "POST", + f"/api/v1/registration-sessions/{registration_session_id}/jobs/{job_uid}/view-edges", + {"view_edges": view_edges}, + auth_header_value=self._registration_session_auth_header(), + allow_auth_fallback=False, + ) + def register_job_inputs_under_registration_session( self, registration_session_id: str, diff --git a/roar/integrations/glaas/registration/artifact.py b/roar/integrations/glaas/registration/artifact.py index 090d9b00..67c6b026 100644 --- a/roar/integrations/glaas/registration/artifact.py +++ b/roar/integrations/glaas/registration/artifact.py @@ -360,6 +360,7 @@ def register_batch_under_registration_session( errors=errors, ) + validation_error_count = len(errors) total_success = 0 total_errors = 0 # Distinct artifacts (one per batch entry), not the server's per-hash @@ -389,25 +390,6 @@ def register_batch_under_registration_session( ) ) - # Backwards compat with glaas instances that pre-date the staged - # artifact endpoint (https://github.com/treqs-inc/glaas-api/pull/50). - # 404 on the very first batch means the server doesn't know the - # endpoint; bail out cleanly so the bearer link path's - # implicit stub-create still works as the legacy fallback. (This is - # exactly the pre-Phase-3 behavior — has the M1 bug, but doesn't - # break the register itself.) - if batch_error and "HTTP 404" in batch_error and batch_idx == 0 and total_success == 0: - self._logger.info( - "Phase 3 endpoint not present on this glaas instance (HTTP 404); " - "falling back to legacy link-implicit artifact creation. " - "Upgrade glaas-api to fix the 0-byte artifact issue." - ) - return ArtifactRegistrationResult( - success_count=0, - error_count=0, - errors=[], - ) - total_success += success_count total_errors += error_count @@ -419,7 +401,7 @@ def register_batch_under_registration_session( return ArtifactRegistrationResult( success_count=distinct_registered, - error_count=total_errors + len(errors), + error_count=total_errors + validation_error_count, errors=errors, ) diff --git a/roar/integrations/glaas/registration/coordinator.py b/roar/integrations/glaas/registration/coordinator.py index b3656da2..11bd9d9e 100644 --- a/roar/integrations/glaas/registration/coordinator.py +++ b/roar/integrations/glaas/registration/coordinator.py @@ -311,6 +311,9 @@ def register_lineage_under_registration_session( links_failed=0, errors=[], already_registered_session_hash=distinct[0], + existing_binding_prepared=bool( + batch_counts.get("existing_binding_prepared", False) + ), ) # Partial overlap (some new + some already elsewhere) or jobs # spanning multiple DAGs — can't form one DAG without a job @@ -367,7 +370,10 @@ def register_lineage_under_registration_session( inputs = self._extract_staged_io_list(job, "_inputs", "_input_hashes") outputs = self._extract_staged_io_list(job, "_outputs", "_output_hashes") - if not inputs and not outputs: + view_edges = job.get("_view_edges") + if not isinstance(view_edges, list): + view_edges = [] + if not inputs and not outputs and not view_edges: continue link_result = self.job_service.link_job_artifacts_under_registration_session( @@ -375,6 +381,7 @@ def register_lineage_under_registration_session( job_uid=remote_job_uid, inputs=inputs, outputs=outputs, + view_edges=view_edges, ) if link_result.success: links_created += link_result.inputs_linked + link_result.outputs_linked diff --git a/roar/integrations/glaas/registration/job.py b/roar/integrations/glaas/registration/job.py index 7ce39729..9704398f 100644 --- a/roar/integrations/glaas/registration/job.py +++ b/roar/integrations/glaas/registration/job.py @@ -631,12 +631,15 @@ def link_job_artifacts_under_registration_session( job_uid: str, inputs: list[dict[str, Any]] | None, outputs: list[dict[str, Any]] | None, + view_edges: list[dict[str, Any]] | None = None, ) -> JobLinkResult: """Link artifacts to a staged job under a remote registration session.""" valid_inputs = self._normalize_link_artifacts(inputs or [], "input") valid_outputs = self._normalize_link_artifacts(outputs or [], "output") - if not valid_inputs and not valid_outputs: + valid_view_edges = [edge for edge in (view_edges or []) if isinstance(edge, dict)] + + if not valid_inputs and not valid_outputs and not valid_view_edges: self._logger.debug( "No staged artifacts to link for registration-session job %s", job_uid, @@ -681,6 +684,15 @@ def link_job_artifacts_under_registration_session( result.get("artifacts_registered", len(batch)) if result else len(batch) ) + if valid_view_edges: + result, error = self.client.register_job_view_edges_under_registration_session( + registration_session_id, + job_uid, + valid_view_edges, + ) + if error: + errors.append(f"view edges: {error}") + if valid_outputs: output_batches = _batch_artifacts(valid_outputs, MAX_ARTIFACTS_PER_REQUEST) for batch_idx, batch in enumerate(output_batches): @@ -718,10 +730,11 @@ def link_job_artifacts_under_registration_session( ) self._logger.debug( - "Linked staged artifacts to registration-session job %s: %d inputs, %d outputs", + "Linked staged artifacts to registration-session job %s: %d inputs, %d outputs, %d view edges", job_uid, inputs_linked, outputs_linked, + len(valid_view_edges), ) return JobLinkResult( success=True, diff --git a/roar/integrations/wandb_trackio.py b/roar/integrations/wandb_trackio.py index c432a8dc..b2420d2b 100644 --- a/roar/integrations/wandb_trackio.py +++ b/roar/integrations/wandb_trackio.py @@ -124,11 +124,25 @@ def _install_trackio_alias(space_id: str) -> bool: def init(*args, **kwargs): for k in _WANDB_ONLY_INIT: kwargs.pop(k, None) + # wandb's `resume` default is None ("do not resume"); trackio accepts only + # "must"/"allow"/"never" and RAISES ValueError on None. Callers that always + # pass the kwarg (lerobot: `resume="must" if cfg.resume else None`) therefore + # die in init(). Map wandb's None onto trackio's "never" — same meaning. + if kwargs.get("resume") is None: + kwargs.pop("resume", None) kwargs.setdefault("space_id", space_id) run = _orig_init(*args, **kwargs) try: if not hasattr(run, "summary"): run.summary = {} + if not hasattr(run, "get_url"): + # wandb code (e.g. lerobot) calls run.get_url(); trackio's Run has + # no such method. run.url exists but returns the bare space id, not + # a URL — aliasing it would stop the crash and publish a broken link + # that still passes a smoke test. COMPOSE the Spaces URL instead; + # space_id and project are both in scope here. + _project = kwargs.get("project") + run.get_url = lambda: f"https://huggingface.co/spaces/{space_id}?project={_project}" except Exception: pass trackio.run = run @@ -140,6 +154,11 @@ def init(*args, **kwargs): def log(*args, **kwargs): kwargs.pop("commit", None) + # wandb's first parameter is NAMED `data`; trackio names it `metrics`. Callers + # using the keyword form (lerobot: `wandb.log(data=batch_data, step=step)`) + # otherwise get TypeError: log() got an unexpected keyword argument 'data'. + if "data" in kwargs and not args: + args = (kwargs.pop("data"),) if args and isinstance(args[0], dict): args = (_to_jsonable(args[0]), *args[1:]) try: @@ -174,9 +193,16 @@ def _f(*a, **k): def _install_noop_wandb() -> None: """Alias ``wandb`` to a silent no-op module so an unmodified repo runs untracked.""" + import importlib.machinery import types mod = types.ModuleType("wandb") + # types.ModuleType leaves __spec__ = None, and importlib.util.find_spec RAISES + # ("wandb.__spec__ is None") rather than returning None on that. accelerate's + # is_wandb_available() calls find_spec at `import accelerate`, so a credential- + # free host (i.e. every cold reproduce host) crashes on import. Give the stub a + # real spec. P0-15. + mod.__spec__ = importlib.machinery.ModuleSpec("wandb", loader=None) def _noop(*a, **k): return None diff --git a/roar/publish_auth.py b/roar/publish_auth.py index a1c8e5d7..16e7291a 100644 --- a/roar/publish_auth.py +++ b/roar/publish_auth.py @@ -2,6 +2,7 @@ import contextvars import json +import os import urllib.error import urllib.request from dataclasses import dataclass @@ -30,6 +31,7 @@ class PublishAuthContext: db_user_id: str | None = None creator_identity: str | None = None ssh_auth_available: bool = False + delegated_auth_available: bool = False # Request-scoped carrier for the explicit --public/--private choice. The publish @@ -127,20 +129,29 @@ def load_publish_auth_context( db_user_id=None, creator_identity=None, ssh_auth_available=False, + delegated_auth_available=False, ) + delegated_auth_available = os.environ.get("ROAR_DELEGATED_AUTH") == "1" access_token = None auth_provider = None user_sub = None db_user_id = None - auth_state = load_auth_state() + # A delegated task deliberately ignores ambient workstation credentials. + # Its loopback broker adds the real upstream authorization out of process. + auth_state = None if delegated_auth_available else load_auth_state() if auth_state is not None: access_token = auth_state.access_token auth_provider = auth_state.provider user_sub = auth_state.user.sub or None db_user_id = auth_state.user.db_user_id - ssh_auth_available = _has_ssh_auth_credentials() + ssh_auth_available = False if delegated_auth_available else _has_ssh_auth_credentials() + + if delegated_auth_available: + auth_provider = "treqs-lineage-task" + user_sub = os.environ.get("ROAR_DELEGATED_USER_SUB") or None + db_user_id = os.environ.get("ROAR_DELEGATED_DB_USER_ID") or None # Proactively renew an expiring/expired bearer so register doesn't ride on a # token `roar whoami` already calls "expired" (which then reads as a bug when @@ -157,8 +168,15 @@ def load_publish_auth_context( access_token = None else: raise PublishAuthError(str(exc)) from exc - binding = None if allow_public_without_binding else _load_repo_binding(start_dir) - repo_scope = load_repo_scope(start_dir) + delegated_scope = _load_delegated_scope() if delegated_auth_available else None + binding = ( + delegated_scope + if delegated_auth_available + else None + if allow_public_without_binding + else _load_repo_binding(start_dir) + ) + repo_scope = None if delegated_auth_available else load_repo_scope(start_dir) # `allow_public_without_binding` permits a *scopeless* public publish, but it # must not discard a **public project scope** — that binding carries the org # attribution (supplier/author) the AI-BOM needs, and a public project's @@ -172,7 +190,8 @@ def load_publish_auth_context( repo_scope and repo_scope.mode == "project" and repo_scope.visibility == "public" ): repo_scope = None - if binding and not access_token and not ssh_auth_available: + has_publish_auth = bool(access_token or ssh_auth_available or delegated_auth_available) + if binding and not has_publish_auth: raise PublishAuthError( "Repo is linked to GLaaS but no global auth state is available. Run `roar login`." ) @@ -183,19 +202,21 @@ def load_publish_auth_context( } if repo_scope.project_id: binding["project_id"] = repo_scope.project_id - if not access_token and not ssh_auth_available: + if not has_publish_auth: raise PublishAuthError( "Repo is linked to GLaaS but no global auth state is available. Run `roar login`." ) - if not binding and not allow_public_without_binding and not access_token: + if not binding and not allow_public_without_binding and not has_publish_auth: raise PublishAuthError( "Private registration requires GLaaS login when no project scope is linked. " "Run `roar login`, use `roar scope use `, or rerun with --public." ) creator_identity = None - if not access_token and allow_public_without_binding: + if delegated_auth_available: + creator_identity = os.environ.get("ROAR_DELEGATED_CREATOR_IDENTITY") or None + elif not access_token and allow_public_without_binding: creator_identity, resolved_db_user_id = _load_authenticated_creator_identity() if resolved_db_user_id and not db_user_id: db_user_id = resolved_db_user_id @@ -205,7 +226,9 @@ def load_publish_auth_context( scope_request = { "owner_id": binding["owner_id"], "owner_type": binding["owner_type"], - "visibility": _scope_visibility(repo_scope, requested_public) or "private", + "visibility": binding.get("visibility") + or _scope_visibility(repo_scope, requested_public) + or "private", } project_id = binding.get("project_id") if project_id: @@ -230,9 +253,26 @@ def load_publish_auth_context( db_user_id=db_user_id, creator_identity=creator_identity, ssh_auth_available=ssh_auth_available, + delegated_auth_available=delegated_auth_available, ) +def _load_delegated_scope() -> dict[str, str]: + values = { + "owner_id": os.environ.get("ROAR_DELEGATED_OWNER_ID", "").strip(), + "owner_type": os.environ.get("ROAR_DELEGATED_OWNER_TYPE", "").strip(), + "project_id": os.environ.get("ROAR_DELEGATED_PROJECT_ID", "").strip(), + "visibility": os.environ.get("ROAR_DELEGATED_VISIBILITY", "").strip(), + } + if ( + not all(values.values()) + or values["owner_type"] not in {"user", "organization"} + or values["visibility"] not in {"public", "private"} + ): + raise PublishAuthError("Delegated GLaaS scope is missing or invalid") + return values + + def resolve_publish_creator_identity(context: PublishAuthContext) -> str: explicit_identity = _optional_string(context.creator_identity) if explicit_identity is not None: diff --git a/rust/tracers/ebpf/userspace/src/daemon.rs b/rust/tracers/ebpf/userspace/src/daemon.rs index b38a4a4c..a372e6fd 100644 --- a/rust/tracers/ebpf/userspace/src/daemon.rs +++ b/rust/tracers/ebpf/userspace/src/daemon.rs @@ -13,7 +13,7 @@ use tracer_runtime::timestamp_now; use crate::events; use crate::ipc::{self, ClientMessage, DaemonMessage}; -use crate::state::{TracerOutput, TracerState}; +use crate::state::{ProcessInfo, TracerOutput, TracerState}; // ── State types ────────────────────────────────────────────────────────────── @@ -46,13 +46,31 @@ impl DaemonState { } } - pub fn register(&mut self, run_id: u64, root_pid: u32) { + pub fn register(&mut self, run_id: u64, root_pid: u32, root_command: Vec) { let mut tracer = TracerState::new(None); tracer.start_time = timestamp_now(); tracer.active_pids.insert(root_pid); // Capture initial process info from /proc (child is SIGSTOP'd but exists) - if let Some(info) = crate::state::capture_process_info(root_pid, None) { + let mut info = crate::state::capture_process_info(root_pid, None); + if !root_command.is_empty() { + // The client told us what it was asked to run, which is + // authoritative: /proc reports the post-exec argv, so a + // `#!/usr/bin/env python3` script reads back as + // `/usr/bin/env python3 ./train.sh` rather than `./train.sh`. + match info.as_mut() { + Some(info) => info.command = root_command, + None => { + info = Some(ProcessInfo { + pid: root_pid, + parent_pid: None, + command: root_command, + env: HashMap::new(), + }) + } + } + } + if let Some(info) = info { tracer.processes.insert(root_pid, info); } @@ -366,9 +384,16 @@ fn handle_client( }; match msg { - ClientMessage::Register { run_id, root_pid } => { + ClientMessage::Register { + run_id, + root_pid, + root_command, + } => { info!("register: run_id={run_id} pid={root_pid}"); - state.lock().unwrap().register(run_id, root_pid); + state + .lock() + .unwrap() + .register(run_id, root_pid, root_command); ipc::send_message(&mut stream, &DaemonMessage::Ack { run_id })?; } ClientMessage::Deregister { run_id } => { @@ -419,7 +444,7 @@ mod tests { #[test] fn test_register_creates_run_state() { let mut state = DaemonState::new(); - state.register(1, 100); + state.register(1, 100, vec![]); assert!(state.runs.contains_key(&1)); let run = &state.runs[&1]; @@ -433,8 +458,8 @@ mod tests { #[test] fn test_deregister_marks_completed_and_keeps_pid_to_run() { let mut state = DaemonState::new(); - state.register(1, 100); - state.register(2, 200); + state.register(1, 100, vec![]); + state.register(2, 200, vec![]); let remaining = state.deregister(1); assert_eq!(remaining, 1); @@ -450,8 +475,8 @@ mod tests { #[test] fn test_get_report_clears_pid_to_run_for_run() { let mut state = DaemonState::new(); - state.register(1, 100); - state.register(2, 200); + state.register(1, 100, vec![]); + state.register(2, 200, vec![]); state.deregister(1); // Still routable until get_report. @@ -466,7 +491,7 @@ mod tests { #[test] fn test_get_report_returns_real_data() { let mut state = DaemonState::new(); - state.register(1, 100); + state.register(1, 100, vec![]); // Simulate some file I/O via the TracerState state @@ -539,7 +564,7 @@ mod tests { #[test] fn test_late_event_after_deregister_is_still_routed() { let mut state = DaemonState::new(); - state.register(1, 100); + state.register(1, 100, vec![]); // Register an open so the FD tracker has a path mapping. state @@ -590,7 +615,7 @@ mod tests { #[test] fn test_event_after_get_report_is_dropped_safely() { let mut state = DaemonState::new(); - state.register(1, 100); + state.register(1, 100, vec![]); state .runs .get_mut(&1) @@ -611,8 +636,8 @@ mod tests { let mut state = DaemonState::new(); assert_eq!(state.active_run_count(), 0); - state.register(1, 100); - state.register(2, 200); + state.register(1, 100, vec![]); + state.register(2, 200, vec![]); assert_eq!(state.active_run_count(), 2); state.deregister(1); @@ -625,9 +650,9 @@ mod tests { #[test] fn test_multiple_registrations_independent() { let mut state = DaemonState::new(); - state.register(1, 100); - state.register(2, 200); - state.register(3, 300); + state.register(1, 100, vec![]); + state.register(2, 200, vec![]); + state.register(3, 300, vec![]); assert_eq!(state.active_run_count(), 3); assert_eq!(state.pid_to_run.get(&100), Some(&1)); @@ -648,8 +673,8 @@ mod tests { #[test] fn test_process_event_routes_to_correct_run() { let mut state = DaemonState::new(); - state.register(1, 100); - state.register(2, 200); + state.register(1, 100, vec![]); + state.register(2, 200, vec![]); // Open a file on run 1's PID (pid=100) state diff --git a/rust/tracers/ebpf/userspace/src/ipc.rs b/rust/tracers/ebpf/userspace/src/ipc.rs index 7d645b46..1e6f696f 100644 --- a/rust/tracers/ebpf/userspace/src/ipc.rs +++ b/rust/tracers/ebpf/userspace/src/ipc.rs @@ -15,9 +15,23 @@ const MAX_PAYLOAD_SIZE: u32 = 16 * 1024 * 1024; #[derive(Serialize, Deserialize, Debug, PartialEq)] #[serde(tag = "type")] pub enum ClientMessage { - Register { run_id: u64, root_pid: u32 }, - Deregister { run_id: u64 }, - GetReport { run_id: u64 }, + Register { + run_id: u64, + root_pid: u32, + /// The command the client was asked to run. Authoritative for the root + /// process, which /proc reports post-exec. Defaulted so a client and + /// daemon of different versions still speak to each other -- the wire + /// format is field-named MessagePack, so the extra key is ignored by an + /// older daemon and absent-means-empty for an older client. + #[serde(default)] + root_command: Vec, + }, + Deregister { + run_id: u64, + }, + GetReport { + run_id: u64, + }, Ping, } @@ -92,6 +106,7 @@ mod tests { ClientMessage::Register { run_id: 42, root_pid: 1234, + root_command: vec!["./train.sh".to_string()], }, ClientMessage::Deregister { run_id: 42 }, ClientMessage::GetReport { run_id: 42 }, @@ -146,6 +161,7 @@ mod tests { let msg = ClientMessage::Register { run_id: 99, root_pid: 5678, + root_command: vec!["./train.sh".to_string()], }; let payload = rmp_serde::to_vec_named(&msg).unwrap(); @@ -167,6 +183,7 @@ mod tests { let msg = ClientMessage::Register { run_id: 123, root_pid: 4567, + root_command: vec!["python".to_string(), "train.py".to_string()], }; send_message(&mut a, &msg).unwrap(); @@ -174,6 +191,36 @@ mod tests { assert_eq!(received, msg); } + /// `roard` is long-lived, so a running daemon can predate the client that + /// connects to it (and vice versa across an upgrade). The wire format is + /// field-named MessagePack, so a Register that omits root_command must + /// still decode -- as empty, which every caller treats as "fall back to + /// /proc" rather than as an empty command line. + #[test] + fn a_register_without_root_command_still_decodes() { + #[derive(Serialize)] + #[serde(tag = "type")] + enum LegacyClientMessage { + Register { run_id: u64, root_pid: u32 }, + } + + let legacy = LegacyClientMessage::Register { + run_id: 7, + root_pid: 4242, + }; + let payload = rmp_serde::to_vec_named(&legacy).unwrap(); + + let decoded: ClientMessage = rmp_serde::from_slice(&payload).unwrap(); + assert_eq!( + decoded, + ClientMessage::Register { + run_id: 7, + root_pid: 4242, + root_command: vec![], + } + ); + } + #[test] fn test_socket_path_format() { let path = socket_path(); diff --git a/rust/tracers/ebpf/userspace/src/main.rs b/rust/tracers/ebpf/userspace/src/main.rs index 30b06aea..1c17db36 100644 --- a/rust/tracers/ebpf/userspace/src/main.rs +++ b/rust/tracers/ebpf/userspace/src/main.rs @@ -201,6 +201,7 @@ fn try_daemon_mode(output_file: &str, command: &[String]) -> Result { &ipc::ClientMessage::Register { run_id, root_pid: child_pid, + root_command: command.to_vec(), }, )?; diff --git a/rust/tracers/preload/src/main.rs b/rust/tracers/preload/src/main.rs index d05fce38..8756264e 100644 --- a/rust/tracers/preload/src/main.rs +++ b/rust/tracers/preload/src/main.rs @@ -398,21 +398,26 @@ impl CollectorState { } else { parent_pid }; - let info = capture_process_info(pid, fallback_parent).unwrap_or_else(|| ProcessInfo { + let mut info = capture_process_info(pid, fallback_parent).unwrap_or_else(|| ProcessInfo { pid, parent_pid: fallback_parent, - command: if pid == self.root_pid { - self.root_command.clone() - } else { - Vec::new() - }, - env: if pid == self.root_pid { - self.root_env.clone() - } else { - HashMap::new() - }, + command: Vec::new(), + env: HashMap::new(), }); + if pid == self.root_pid { + // We launched this process, so its argv is known exactly. Prefer it + // over /proc, which reports what the kernel ran rather than what the + // user asked for: a `#!/usr/bin/env python3` script reads back as + // `/usr/bin/env python3 ./train.sh`, and is empty altogether if the + // process exits before the read. Descendants have no such source and + // keep using /proc. + info.command = self.root_command.clone(); + if info.env.is_empty() { + info.env = self.root_env.clone(); + } + } + self.processes.insert(pid, info); } @@ -1030,6 +1035,34 @@ mod tests { use super::*; use tracer_schema::FileRecord; + /// The root's argv is what we were asked to run, not what /proc reports. + /// Using our own live pid as the root makes the two differ observably: the + /// launcher command below is nothing like this test binary's real argv. + #[test] + fn the_root_command_comes_from_the_launcher_not_proc() { + let launched = vec!["./train.sh".to_string()]; + let mut state = CollectorState::new(std::process::id(), launched.clone()); + + state.ensure_process(std::process::id()); + + let root = state.processes.get(&std::process::id()).unwrap(); + assert_eq!(root.command, launched); + // /proc was still consulted for everything else. + assert!(!root.env.is_empty(), "env should still come from /proc"); + } + + /// Descendants have no launcher-supplied argv, so they keep using /proc. + #[test] + fn a_descendant_command_still_comes_from_proc() { + let mut state = CollectorState::new(1, vec!["./train.sh".to_string()]); + + state.ensure_process(std::process::id()); + + let child = state.processes.get(&std::process::id()).unwrap(); + assert_ne!(child.command, vec!["./train.sh".to_string()]); + assert!(!child.command.is_empty()); + } + fn written_record(path: &str) -> FileRecord { FileRecord { path: path.to_string(), @@ -1076,7 +1109,10 @@ mod tests { state.reconcile_renamed_outputs(&mut summary); - assert_eq!(summary.files[0].path, final_str, "record rewritten to final name"); + assert_eq!( + summary.files[0].path, final_str, + "record rewritten to final name" + ); assert!(summary.written_files.contains(&final_str)); assert!(!summary.written_files.contains(&temp_str)); let _ = fs::remove_dir_all(&dir); @@ -1108,7 +1144,10 @@ mod tests { }; state.reconcile_renamed_outputs(&mut summary); - assert_eq!(summary.files[0].path, temp_str, "deleted file path unchanged"); + assert_eq!( + summary.files[0].path, temp_str, + "deleted file path unchanged" + ); let _ = fs::remove_dir_all(&dir); } diff --git a/rust/tracers/ptrace/src/main.rs b/rust/tracers/ptrace/src/main.rs index 0185acee..2875e300 100644 --- a/rust/tracers/ptrace/src/main.rs +++ b/rust/tracers/ptrace/src/main.rs @@ -108,10 +108,16 @@ struct TracerState { // CWD cache per PID cwd_cache: HashMap, + + // The command the launcher was asked to run, and the pid it became. This + // is authoritative for the root process: /proc reports what the kernel ran + // rather than what the user asked for. + root_pid: Option, + root_command: Vec, } impl TracerState { - fn new() -> Self { + fn new(root_command: Vec) -> Self { TracerState { processes: HashMap::new(), fd_tracker: FdTracker::new(None), @@ -126,6 +132,8 @@ impl TracerState { pending_fchdirs: HashMap::new(), active_pids: HashSet::new(), cwd_cache: HashMap::new(), + root_pid: None, + root_command, } } } @@ -170,9 +178,26 @@ fn capture_process_info(pid: Pid, state: &mut TracerState, parent_pid: Option info, + // The root's argv came from the launcher, so it is worth recording even + // when /proc could not be read at all. + None if is_root => ProcessInfo { + pid: pid_raw as u32, + parent_pid, + command: Vec::new(), + env: HashMap::new(), + }, + None => return, + }; + if is_root && !state.root_command.is_empty() { + // Authoritative: we launched it. /proc would report the post-exec argv, + // so a `#!/usr/bin/env python3` script reads back as + // `/usr/bin/env python3 ./train.sh` rather than `./train.sh`. + info.command = state.root_command.clone(); } + state.processes.insert(pid_raw, info); } // ============================================================================= @@ -781,7 +806,7 @@ fn run_preflight(json_output: bool, command: Option<&str>) -> i32 { fn run_tracer(command: Vec, output_file: &str) -> i32 { let start_time = timestamp_now(); - let mut state = TracerState::new(); + let mut state = TracerState::new(command.clone()); // Fork and trace match unsafe { fork() } { @@ -807,6 +832,7 @@ fn run_tracer(command: Vec, output_file: &str) -> i32 { // Parent: wait for child to stop at exec, then trace let child_pid = child.as_raw(); state.active_pids.insert(child_pid); + state.root_pid = Some(child_pid); // Wait for initial stop match waitpid(child, None) { diff --git a/scripts/build_wheel_with_bins.sh b/scripts/build_wheel_with_bins.sh index 21d36984..8658b984 100755 --- a/scripts/build_wheel_with_bins.sh +++ b/scripts/build_wheel_with_bins.sh @@ -139,6 +139,20 @@ resolve_built_artifact() { } build_python_wheel() { + if [[ "$(uname -s)" == "Linux" ]]; then + echo "▶ Building portable manylinux_2_17 wheel with maturin and Zig..." + ( + cd "$ROOT_DIR" + uvx --from 'maturin[zig]' maturin build \ + --release \ + --zig \ + --compatibility manylinux_2_17 \ + --manifest-path rust/crates/artifact-hash-py/Cargo.toml \ + --out "$OUT_DIR" + ) + return + fi + if command -v uv >/dev/null 2>&1; then echo "▶ Building wheel with uv..." uv build --wheel --out-dir "$OUT_DIR" diff --git a/scripts/ci/verify_wheel_contents.py b/scripts/ci/verify_wheel_contents.py index 2a37073d..1d189f01 100644 --- a/scripts/ci/verify_wheel_contents.py +++ b/scripts/ci/verify_wheel_contents.py @@ -57,11 +57,13 @@ def main() -> None: if missing_bins: raise SystemExit(f"Missing binaries in wheel: {missing_bins}") - has_native = any( - name.startswith("roar/_hash_native") - and (name.endswith(".so") or name.endswith(".pyd") or name.endswith(".dylib")) + native_extensions = { + name for name in names - ) + if name.startswith("roar/_hash_native") + and (name.endswith(".so") or name.endswith(".pyd") or name.endswith(".dylib")) + } + has_native = bool(native_extensions) if not has_native: raise SystemExit("Missing native hash extension in wheel (roar/_hash_native*)") @@ -74,9 +76,10 @@ def main() -> None: raise SystemExit("Missing preload interposer library in wheel (roar/bin/libroar*_preload*)") if platform == "linux": - _verify_linux_glibc_floor(wheel, names, required_bins) + linux_elf_members = required_bins | native_extensions + _verify_linux_glibc_floor(wheel, names, linux_elf_members) if expected_arch is not None: - _verify_linux_bin_arch(wheel, names, required_bins, expected_arch) + _verify_linux_bin_arch(wheel, names, linux_elf_members, expected_arch) print(f"Verified wheel contents: {wheel}") diff --git a/tests/application/publish/test_collection.py b/tests/application/publish/test_collection.py index bc1428ca..ea588617 100644 --- a/tests/application/publish/test_collection.py +++ b/tests/application/publish/test_collection.py @@ -27,6 +27,42 @@ def test_collect_register_lineage_returns_missing_file_error(tmp_path: Path) -> assert error == "File not found: missing.csv" +def test_collect_register_lineage_selects_active_session_without_a_prebootstrap_hash( + tmp_path: Path, +) -> None: + collector = MagicMock() + collector.collect_session.return_value = LineageData( + jobs=[{"job_uid": "job-active"}], + artifacts=[], + artifact_hashes=set(), + pipeline={"id": 17}, + ) + with patch("roar.application.publish.collection.create_database_context") as mock_ctx: + db_ctx = MagicMock() + db_ctx.__enter__ = MagicMock(return_value=db_ctx) + db_ctx.__exit__ = MagicMock(return_value=None) + db_ctx.sessions.get_active.return_value = {"id": 17} + mock_ctx.return_value = db_ctx + + collected, error = collect_register_lineage( + target=ResolvedRegisterTarget(kind="active_session", value=""), + roar_dir=tmp_path / ".roar", + cwd=tmp_path, + lineage_collector=collector, + session_service=MagicMock(), + logger=MagicMock(), + ) + + assert error is None + assert collected == CollectedRegisterLineage( + lineage=collector.collect_session.return_value, + session_id=17, + artifact_hash="", + session_hash_override=None, + ) + collector.collect_session.assert_called_once_with(17, tmp_path / ".roar") + + def test_collect_register_lineage_resolves_s3_artifact_by_tracked_path(tmp_path: Path) -> None: collector = MagicMock() collector.collect.return_value = LineageData( diff --git a/tests/application/publish/test_put_preparation.py b/tests/application/publish/test_put_preparation.py index 619f2708..0c1b9543 100644 --- a/tests/application/publish/test_put_preparation.py +++ b/tests/application/publish/test_put_preparation.py @@ -4,9 +4,18 @@ from unittest.mock import MagicMock, patch import pytest +from sqlalchemy import text -from roar.application.publish.put_preparation import PreparedPutExecution, prepare_put_execution +from roar.application.publish.put_execution import PutService +from roar.application.publish.put_preparation import ( + PreparedPutExecution, + complete_delegated_put_operation, + prepare_put_execution, +) from roar.core.interfaces.registration import GitContext +from roar.db.context import create_database_context +from roar.db.hashing import hash_files_blake3 +from roar.integrations.storage import MemoryBackend def test_prepare_put_execution_requires_active_session(tmp_path: Path) -> None: @@ -34,6 +43,7 @@ def test_prepare_put_execution_builds_session_git_and_source_plan(tmp_path: Path db_ctx = MagicMock() db_ctx.sessions.get_active.return_value = {"id": 7} runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" prepared_session = MagicMock( session_hash="session-hash", session_url="https://glaas/session", @@ -51,7 +61,7 @@ def test_prepare_put_execution_builds_session_git_and_source_plan(tmp_path: Path patch( "roar.application.publish.put_preparation.prepare_publish_session", return_value=prepared_session, - ), + ) as prepare_session, patch( "roar.application.publish.put_preparation.infer_publish_dataset_identifiers", return_value=[], @@ -81,8 +91,65 @@ def test_prepare_put_execution_builds_session_git_and_source_plan(tmp_path: Path resolved_sources=prepared.resolved_sources, destination_type="memory", composite_source_type=None, + source_hashes=prepared.source_hashes, ) assert [item.path for item in prepared.resolved_sources] == [model.resolve()] + assert prepared.source_hashes[str(model.resolve())] + call = prepare_session.call_args.kwargs + assert call["operation_kind"] == "put" + assert len(call["operation_fingerprint"]) == 64 + + +def test_prepare_put_execution_fingerprints_source_content(tmp_path: Path) -> None: + model = tmp_path / "model.pt" + model.write_bytes(b"model-v1") + db_ctx = MagicMock() + db_ctx.sessions.get_active.return_value = {"id": 7} + runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" + prepared_session = MagicMock( + session_hash="session-hash", + session_url=None, + registration_session_id=None, + registration_session_mode=None, + ) + + with ( + patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=GitContext(repo="repo", branch="main", commit="deadbeef"), + ), + patch( + "roar.application.publish.put_preparation.prepare_publish_session", + return_value=prepared_session, + ) as prepare_session, + ): + prepare_put_execution( + db_ctx=db_ctx, + runtime=runtime, + roar_dir=tmp_path / ".roar", + repo_root=tmp_path, + sources=["model.pt"], + destination="memory://bucket/prefix", + git_commit="deadbeef", + logger=MagicMock(), + ) + first_fingerprint = prepare_session.call_args.kwargs["operation_fingerprint"] + + model.write_bytes(b"model-v2") + prepare_put_execution( + db_ctx=db_ctx, + runtime=runtime, + roar_dir=tmp_path / ".roar", + repo_root=tmp_path, + sources=["model.pt"], + destination="memory://bucket/prefix", + git_commit="deadbeef", + logger=MagicMock(), + ) + second_fingerprint = prepare_session.call_args.kwargs["operation_fingerprint"] + + assert first_fingerprint != second_fingerprint def test_prepare_put_execution_propagates_missing_source(tmp_path: Path) -> None: @@ -116,3 +183,459 @@ def test_prepare_put_execution_propagates_missing_source(tmp_path: Path) -> None git_commit="deadbeef", logger=MagicMock(), ) + + +def test_delegated_put_reuses_pending_retry_then_advances_identical_operation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _set_delegated_task(monkeypatch) + model = tmp_path / "model.pt" + model.write_bytes(b"model") + roar_dir = tmp_path / ".roar" + runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" + git_context = GitContext(repo="repo", branch="main", commit="deadbeef") + + with create_database_context(roar_dir) as db_ctx: + db_ctx.sessions.get_or_create_active() + db_ctx.commit() + with ( + patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=git_context, + ), + patch( + "roar.application.publish.put_preparation.prepare_publish_session", + return_value=MagicMock( + session_hash="session-hash", + session_url=None, + registration_session_id="registration-session", + registration_session_mode="delegated", + registration_session_status="active", + ), + ) as prepare_session, + ): + first = _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + first_fingerprint = prepare_session.call_args.kwargs["operation_fingerprint"] + retry = _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + retry_fingerprint = prepare_session.call_args.kwargs["operation_fingerprint"] + + assert first.delegated_put_operation is not None + assert retry.delegated_put_operation == first.delegated_put_operation + assert retry_fingerprint == first_fingerprint + + complete_delegated_put_operation(db_ctx, retry.delegated_put_operation) + second = _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + second_fingerprint = prepare_session.call_args.kwargs["operation_fingerprint"] + + assert second.delegated_put_operation is not None + assert second.delegated_put_operation.ordinal == 2 + assert second_fingerprint != first_fingerprint + + +def test_delegated_put_rejects_changed_git_context_while_retry_is_pending( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _set_delegated_task(monkeypatch) + (tmp_path / "model.pt").write_bytes(b"model") + roar_dir = tmp_path / ".roar" + runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" + + with create_database_context(roar_dir) as db_ctx: + db_ctx.sessions.get_or_create_active() + db_ctx.commit() + with patch( + "roar.application.publish.put_preparation.prepare_publish_session", + return_value=MagicMock( + session_hash="session-hash", + session_url=None, + registration_session_id="registration-session", + registration_session_mode="delegated", + registration_session_status="active", + ), + ): + with patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=GitContext(repo="repo", branch="main", commit="first"), + ): + _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + with ( + patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=GitContext(repo="repo", branch="feature", commit="second"), + ), + pytest.raises(ValueError, match="different delegated put operation"), + ): + _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + + +def test_delegated_put_rejects_changed_lineage_while_retry_is_pending( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _set_delegated_task(monkeypatch) + (tmp_path / "model.pt").write_bytes(b"model") + roar_dir = tmp_path / ".roar" + runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" + git_context = GitContext(repo="repo", branch="main", commit="deadbeef") + + with create_database_context(roar_dir) as db_ctx: + session_id = db_ctx.sessions.get_or_create_active() + db_ctx.commit() + with ( + patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=git_context, + ), + patch( + "roar.application.publish.put_preparation.prepare_publish_session", + return_value=MagicMock( + session_hash="session-hash", + session_url=None, + registration_session_id="registration-session", + registration_session_mode="delegated", + registration_session_status="active", + ), + ), + ): + _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + _record_model_producer(db_ctx, session_id, tmp_path / "model.pt") + db_ctx.commit() + + with pytest.raises(ValueError, match="different delegated put operation"): + _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + + +def test_delegated_put_ignores_unrelated_active_session_jobs_on_retry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _set_delegated_task(monkeypatch) + (tmp_path / "model.pt").write_bytes(b"model") + roar_dir = tmp_path / ".roar" + runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" + git_context = GitContext(repo="repo", branch="main", commit="deadbeef") + + with create_database_context(roar_dir) as db_ctx: + session_id = db_ctx.sessions.get_or_create_active() + db_ctx.commit() + with ( + patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=git_context, + ), + patch( + "roar.application.publish.put_preparation.prepare_publish_session", + return_value=MagicMock( + session_hash="session-hash", + session_url=None, + registration_session_id="registration-session", + registration_session_mode="delegated", + registration_session_status="active", + ), + ), + ): + first = _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + db_ctx.session.execute( + text( + """ + INSERT INTO jobs (timestamp, command, session_id, step_number) + VALUES (1, 'python unrelated.py', :session_id, 1) + """ + ), + {"session_id": session_id}, + ) + db_ctx.commit() + retry = _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + + assert retry.delegated_put_operation == first.delegated_put_operation + + +def test_delegated_put_retry_excludes_its_persisted_sink_and_recovers_closed_receipt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _set_delegated_task(monkeypatch) + (tmp_path / "model.pt").write_bytes(b"model") + roar_dir = tmp_path / ".roar" + runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" + git_context = GitContext(repo="repo", branch="main", commit="deadbeef") + + with create_database_context(roar_dir) as db_ctx: + session_id = db_ctx.sessions.get_or_create_active() + db_ctx.commit() + with ( + patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=git_context, + ), + patch( + "roar.application.publish.put_preparation.prepare_publish_session", + side_effect=[ + MagicMock( + session_hash="provisional-hash", + session_url=None, + registration_session_id="registration-session", + registration_session_mode="delegated", + registration_session_status="active", + ), + MagicMock( + session_hash="authoritative-hash", + session_url="https://glaas.example/dag/authoritative-hash", + registration_session_id="registration-session", + registration_session_mode="delegated", + registration_session_status="closed", + ), + ], + ), + ): + first = _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + assert first.delegated_put_operation is not None + put_job_uid = first.delegated_put_operation.put_job_uid + put_job_id, created_uid = db_ctx.jobs.create( + command="roar put model.pt memory://bucket/prefix", + timestamp=1, + job_uid=put_job_uid, + session_id=session_id, + step_number=1, + job_type="put", + ) + assert created_uid == put_job_uid + db_ctx.session.execute( + text( + """ + INSERT INTO artifacts (id, size, first_seen_at, first_seen_path) + VALUES ('put-artifact', 5, 1, 'model.pt') + """ + ) + ) + db_ctx.jobs.add_input(put_job_id, "put-artifact", "model.pt") + db_ctx.commit() + + retry = _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + assert retry.delegated_put_operation == first.delegated_put_operation + service = PutService( + db_context=db_ctx, + backend=MemoryBackend(bucket="bucket", prefix="prefix"), + destination="memory://bucket/prefix", + repo_root=tmp_path, + ) + result = service.put_prepared( + prepared=retry, + sources=["model.pt"], + message="retry", + ) + assert result.success is True + assert result.session_hash == "authoritative-hash" + complete_delegated_put_operation(db_ctx, retry.delegated_put_operation) + + row = ( + db_ctx.session.execute( + text( + """ + SELECT status, ordinal, put_job_uid + FROM delegated_put_operations + WHERE task_identity = :task_identity AND session_id = :session_id + """ + ), + { + "task_identity": retry.delegated_put_operation.task_identity, + "session_id": session_id, + }, + ) + .mappings() + .one() + ) + put_job_count = db_ctx.session.execute( + text("SELECT COUNT(*) FROM jobs WHERE job_uid = :job_uid"), + {"job_uid": put_job_uid}, + ).scalar_one() + + assert dict(row) == {"status": "completed", "ordinal": 1, "put_job_uid": put_job_uid} + assert put_job_count == 1 + + +@pytest.mark.parametrize( + ("mutation_sql", "params"), + [ + ("UPDATE jobs SET job_type = 'ray_task' WHERE job_uid = 'upstream'", {}), + ("UPDATE jobs SET parent_job_uid = 'parent' WHERE job_uid = 'upstream'", {}), + ( + "UPDATE jobs SET metadata = :metadata WHERE job_uid = 'upstream'", + {"metadata": '{"changed":true}'}, + ), + ( + "UPDATE job_outputs SET byte_ranges = '[[0,4]]' WHERE job_id = :job_id", + {"job_id": 1}, + ), + ], +) +def test_delegated_put_rejects_emitted_lineage_contract_changes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + mutation_sql: str, + params: dict[str, object], +) -> None: + _set_delegated_task(monkeypatch) + (tmp_path / "model.pt").write_bytes(b"model") + roar_dir = tmp_path / ".roar" + runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" + git_context = GitContext(repo="repo", branch="main", commit="deadbeef") + + with create_database_context(roar_dir) as db_ctx: + session_id = db_ctx.sessions.get_or_create_active() + job_id = _record_model_producer(db_ctx, session_id, tmp_path / "model.pt") + db_ctx.commit() + with ( + patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=git_context, + ), + patch( + "roar.application.publish.put_preparation.prepare_publish_session", + return_value=MagicMock( + session_hash="session-hash", + session_url=None, + registration_session_id="registration-session", + registration_session_mode="delegated", + registration_session_status="active", + ), + ), + ): + _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + bound_params = {**params, "job_id": job_id} + db_ctx.session.execute(text(mutation_sql), bound_params) + db_ctx.commit() + with pytest.raises(ValueError, match="different delegated put operation"): + _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + + +def test_delegated_put_rejects_changed_producer_from_an_earlier_session( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _set_delegated_task(monkeypatch) + model = tmp_path / "model.pt" + model.write_bytes(b"model") + model_digest = hash_files_blake3([model])[str(model)] + roar_dir = tmp_path / ".roar" + runtime = MagicMock() + runtime.session_service.compute_session_hash.return_value = "local-session-hash" + git_context = GitContext(repo="repo", branch="main", commit="deadbeef") + + with create_database_context(roar_dir) as db_ctx: + producer_session_id = db_ctx.sessions.get_or_create_active() + producer_job_id, _ = db_ctx.jobs.create( + command="python produce.py", + timestamp=1, + job_uid="earlier-producer", + session_id=producer_session_id, + step_number=1, + metadata="{}", + ) + db_ctx.session.execute( + text( + """ + INSERT INTO artifacts (id, size, first_seen_at, first_seen_path) + VALUES ('model-artifact', 5, 1, 'model.pt') + """ + ) + ) + db_ctx.session.execute( + text( + """ + INSERT INTO artifact_hashes (artifact_id, algorithm, digest) + VALUES ('model-artifact', 'blake3', :digest) + """ + ), + {"digest": model_digest}, + ) + db_ctx.jobs.add_output(producer_job_id, "model-artifact", "model.pt") + active_session_id = db_ctx.sessions.create(make_active=True) + db_ctx.commit() + + with ( + patch( + "roar.application.publish.put_preparation.resolve_roar_git_context", + return_value=git_context, + ), + patch( + "roar.application.publish.put_preparation.prepare_publish_session", + return_value=MagicMock( + session_hash="session-hash", + session_url=None, + registration_session_id="registration-session", + registration_session_mode="delegated", + registration_session_status="active", + ), + ), + ): + prepared = _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + assert prepared.session_id == active_session_id + db_ctx.session.execute( + text("UPDATE jobs SET metadata = :metadata WHERE id = :job_id"), + {"job_id": producer_job_id, "metadata": '{"changed":true}'}, + ) + db_ctx.commit() + + with pytest.raises(ValueError, match="different delegated put operation"): + _prepare_model_put(db_ctx, runtime, roar_dir, tmp_path) + + +def _set_delegated_task(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ROAR_DELEGATED_JOB_ID", "job-1") + monkeypatch.setenv("ROAR_DELEGATED_EXECUTION_ATTEMPT_ID", "attempt-1") + monkeypatch.setenv("ROAR_DELEGATED_TASK_ID", "task-1") + + +def _record_model_producer(db_ctx, session_id: int, model: Path) -> int: + digest = hash_files_blake3([model])[str(model)] + artifact_id = f"model-artifact-{session_id}" + job_id, _ = db_ctx.jobs.create( + command="python upstream.py", + timestamp=1, + job_uid="upstream", + session_id=session_id, + step_number=1, + metadata="{}", + ) + db_ctx.session.execute( + text( + """ + INSERT INTO artifacts (id, size, first_seen_at, first_seen_path) + VALUES (:artifact_id, 5, 1, 'model.pt') + """ + ), + {"artifact_id": artifact_id}, + ) + db_ctx.session.execute( + text( + """ + INSERT INTO artifact_hashes (artifact_id, algorithm, digest) + VALUES (:artifact_id, 'blake3', :digest) + """ + ), + {"artifact_id": artifact_id, "digest": digest}, + ) + db_ctx.jobs.add_output(job_id, artifact_id, "model.pt") + return job_id + + +def _prepare_model_put(db_ctx, runtime, roar_dir: Path, repo_root: Path) -> PreparedPutExecution: + return prepare_put_execution( + db_ctx=db_ctx, + runtime=runtime, + roar_dir=roar_dir, + repo_root=repo_root, + sources=["model.pt"], + destination="memory://bucket/prefix", + git_commit="deadbeef", + logger=MagicMock(), + ) diff --git a/tests/application/publish/test_registration.py b/tests/application/publish/test_registration.py index 4be7ecec..e1c0128f 100644 --- a/tests/application/publish/test_registration.py +++ b/tests/application/publish/test_registration.py @@ -201,6 +201,35 @@ def test_sync_publish_labels_appends_error_when_sync_fails() -> None: assert errors == ["Label sync failed: permission denied"] +def test_sync_publish_labels_uses_registration_session_scoped_route() -> None: + client = MagicMock() + client.sync_labels_under_registration_session.return_value = ( + {"processed": 1, "created": 1}, + None, + ) + + with patch( + "roar.application.publish.registration.collect_label_sync_payloads", + return_value=[{"entity_type": "dag", "session_hash": "session-hash"}], + ): + sync_publish_labels( + glaas_client=client, + db_ctx=MagicMock(), + session_id=7, + session_hash="session-hash", + jobs=[{"job_uid": "job-1"}], + artifacts=[], + errors=[], + registration_session_id="reg-session-123", + ) + + client.sync_labels_under_registration_session.assert_called_once_with( + "reg-session-123", + [{"entity_type": "dag", "session_hash": "session-hash"}], + ) + client.sync_labels.assert_not_called() + + def test_sync_publish_labels_skips_empty_payloads() -> None: client = MagicMock() diff --git a/tests/application/publish/test_service.py b/tests/application/publish/test_service.py index 55809746..8fe2c7a9 100644 --- a/tests/application/publish/test_service.py +++ b/tests/application/publish/test_service.py @@ -390,6 +390,9 @@ def test_put_artifacts_continues_when_git_preflight_warns(tmp_path: Path) -> Non "roar.application.publish.service.finalize_put_git", return_value=(None, []), ), + patch( + "roar.application.publish.service.complete_delegated_put_operation" + ) as complete_operation, ): mock_put_cls.return_value.put_prepared.return_value = put_result @@ -414,6 +417,7 @@ def test_put_artifacts_continues_when_git_preflight_warns(tmp_path: Path) -> Non reproducible=False, commit_on_remote=False, ) + complete_operation.assert_called_once_with(db_ctx, prepared.delegated_put_operation) def test_put_artifacts_returns_preparation_error_before_service(tmp_path: Path) -> None: diff --git a/tests/application/publish/test_session.py b/tests/application/publish/test_session.py index 10ed7660..2d55f832 100644 --- a/tests/application/publish/test_session.py +++ b/tests/application/publish/test_session.py @@ -5,7 +5,11 @@ import pytest -from roar.application.publish.session import PreparedPublishSession, prepare_publish_session +from roar.application.publish.session import ( + PreparedPublishSession, + delegated_client_session_id, + prepare_publish_session, +) from roar.core.interfaces.lineage import LineageData from roar.core.interfaces.registration import GitContext, SessionRegistrationResult @@ -31,6 +35,32 @@ def _lineage() -> LineageData: ) +def test_delegated_client_session_id_is_operation_scoped(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ROAR_DELEGATED_JOB_ID", "job-1") + monkeypatch.setenv("ROAR_DELEGATED_EXECUTION_ATTEMPT_ID", "attempt-1") + monkeypatch.setenv("ROAR_DELEGATED_TASK_ID", "task-1") + + register_id = delegated_client_session_id( + operation_kind="register", + operation_fingerprint="lineage-a", + ) + + assert register_id == delegated_client_session_id( + operation_kind="register", + operation_fingerprint="lineage-a", + ) + assert register_id != delegated_client_session_id( + operation_kind="register", + operation_fingerprint="lineage-b", + ) + assert register_id != delegated_client_session_id( + operation_kind="put", + operation_fingerprint="lineage-a", + ) + assert register_id is not None + assert register_id.startswith("roar-delegated-v2-") + + def test_prepare_publish_session_computes_hash_without_registering(tmp_path: Path) -> None: glaas_client = MagicMock() session_service = MagicMock() @@ -234,6 +264,50 @@ def test_prepare_publish_session_creates_registration_session_with_scoped_ssh_on session_service.register.assert_not_called() +def test_prepare_publish_session_creates_registration_session_with_delegated_auth( + tmp_path: Path, +) -> None: + glaas_client = MagicMock() + glaas_client.publish_auth.access_token = None + glaas_client.publish_auth.scope_request = { + "owner_id": "owner-123", + "owner_type": "organization", + "project_id": "proj-123", + "visibility": "private", + } + glaas_client.publish_auth.ssh_auth_available = False + glaas_client.publish_auth.delegated_auth_available = True + session_service = MagicMock() + session_service.compute_session_hash.return_value = "session-hash" + session_service.create_registration_session.return_value = SessionRegistrationResult( + success=True, + session_hash="session-hash", + session_url=None, + registration_session_id="reg-session-delegated-123", + ) + + result = prepare_publish_session( + glaas_client=glaas_client, + session_service=session_service, + roar_dir=tmp_path / ".roar", + session_id=7, + git_context=_git_context(), + logger=MagicMock(), + register_with_glaas=True, + ) + + assert result == PreparedPublishSession( + session_hash="session-hash", + session_url=None, + registration_session_id="reg-session-delegated-123", + ) + session_service.create_registration_session.assert_called_once_with( + client_session_id=None, + mode=None, + ) + session_service.register.assert_not_called() + + def test_prepare_publish_session_uses_anonymous_public_registration_sessions_when_supported( tmp_path: Path, ) -> None: diff --git a/tests/application/publish/test_staged_lineage_counts.py b/tests/application/publish/test_staged_lineage_counts.py new file mode 100644 index 00000000..798e508a --- /dev/null +++ b/tests/application/publish/test_staged_lineage_counts.py @@ -0,0 +1,57 @@ +"""P0-22 (Option B): roar's finalize count AND canonical session hash must dedup +job edges by CONTENT hash, matching glaas's (job_id, artifact_hash) storage key — +so a workload writing identical bytes to two names (timm os.link last/best/ +checkpoint) doesn't 400 finalize or diverge from glaas's published hash. +""" + +from __future__ import annotations + +from roar.application.publish.session import ( + _dedup_edges_by_hash, + build_staged_lineage_counts, +) + + +def _job(inputs=(), outputs=()): + return {"_inputs": list(inputs), "_outputs": list(outputs)} + + +def test_hardlink_duplicate_outputs_collapse_to_one_content(): + dup = [ + {"hash": "abc", "path": "last.pth.tar", "byte_ranges": None}, + {"hash": "abc", "path": "checkpoint-9.pth.tar", "byte_ranges": None}, + {"hash": "abc", "path": "model_best.pth.tar", "byte_ranges": None}, + {"hash": "def", "path": "config.json", "byte_ranges": None}, + ] + assert build_staged_lineage_counts([_job(outputs=dup)])["outputs"] == 2 + + +def test_no_duplicates_is_a_noop_equal_to_path_count(): + outs = [{"hash": h, "path": h, "byte_ranges": None} for h in ("a", "b", "c")] + assert build_staged_lineage_counts([_job(outputs=outs)])["outputs"] == 3 + + +def test_dedup_keeps_smallest_path_deterministically(): + # matches glaas skipDuplicates when staged in sorted order; stable representative + edges = [ + {"hash": "x", "path": "zzz"}, + {"hash": "x", "path": "aaa"}, + {"hash": "x", "path": "mmm"}, + ] + kept = _dedup_edges_by_hash(edges) + assert len(kept) == 1 and kept[0]["path"] == "aaa" + + +def test_dedup_is_by_hash_not_byte_ranges(): + # glaas's key ignores byte_ranges, so same hash + different ranges still collapses + edges = [ + {"hash": "z", "path": "d", "byte_ranges": [[0, 100]]}, + {"hash": "z", "path": "d", "byte_ranges": [[100, 200]]}, + ] + assert len(_dedup_edges_by_hash(edges)) == 1 + + +def test_edges_without_a_hash_are_dropped(): + assert ( + build_staged_lineage_counts([_job(outputs=[{"path": "x"}, {"hash": ""}])])["outputs"] == 0 + ) diff --git a/tests/application/reproducibility/test_report.py b/tests/application/reproducibility/test_report.py index cbf7b853..ffe4b8c0 100644 --- a/tests/application/reproducibility/test_report.py +++ b/tests/application/reproducibility/test_report.py @@ -9,10 +9,33 @@ build_report, is_shareable_remote, render_report, + runtime_captured, untracked_artifact_dirs, ) +class _Pipeline: + def __init__(self, metadata): + self.build_steps = [] + self.run_steps = [{"metadata": metadata}] + + +def test_runtime_capture_rejects_explicitly_missing_python_injection(): + pipeline = _Pipeline( + {"runtime": {"python": {"version": "3.12.1"}}, "python_capture": "missing"} + ) + assert runtime_captured(pipeline) is False + + +def test_runtime_capture_accepts_complete_and_legacy_lineage(): + complete = _Pipeline( + {"runtime": {"python": {"version": "3.12.1"}}, "python_capture": "complete"} + ) + legacy = _Pipeline({"runtime": {"python": {"version": "3.12.1"}}}) + assert runtime_captured(complete) is True + assert runtime_captured(legacy) is True + + def _full_report(**overrides): # A register/put-style report: every fact supplied, so all checks render # (including the receipt-only `paths_tracked` and `on_glaas`). diff --git a/tests/execution/runtime/test_inject_log_merge.py b/tests/execution/runtime/test_inject_log_merge.py new file mode 100644 index 00000000..6c73f844 --- /dev/null +++ b/tests/execution/runtime/test_inject_log_merge.py @@ -0,0 +1,265 @@ +"""P0-9: in a traced process tree, every process inherits one ROAR_LOG_FILE and +used to open it "w" — so a multiprocessing worker (litdata/DataLoader/HF datasets +num_proc/torchrun) could truncate the workload's record and be the one that +survived. write_log now writes a per-PID shard and merge_inject_logs unions them, +recovering the full workload instead of whichever process wrote last. +""" + +from __future__ import annotations + +import builtins +import json +import multiprocessing +import os + +from roar.execution.runtime.inject.tracker import ( + RuntimeInjectionTracker, + merge_inject_logs, +) + + +class _FakeController: + def handle_import(self, module_name, module): + return None + + +class _FakeEnviron(dict): + """A dict that tolerates the attribute patching ``install()`` performs.""" + + +def _tracker(log_path): + return RuntimeInjectionTracker( + {"ROAR_LOG_FILE": str(log_path)}, + _FakeController(), + log_file=str(log_path), + inject_dir=str(log_path.parent / "inject"), + ) + + +def _shard(base, pid, data): + (base.parent / f"{base.name}.{pid}").write_text(json.dumps(data), encoding="utf-8") + + +def _record_fork_only_import(tracker): + tracker.imported_modules.add("fork_only_dependency") + + +def _record_then_force_exit(tracker): + # Recorded after fork, so only an exit hook could capture it -- and + # os._exit runs none. + tracker.imported_modules.add("fork_only_dependency") + os._exit(0) + + +def _installable_tracker(log_path): + """A tracker whose environ tolerates ``install()``'s attribute patching.""" + return RuntimeInjectionTracker( + _FakeEnviron({"ROAR_LOG_FILE": str(log_path)}), + _FakeController(), + log_file=str(log_path), + inject_dir=str(log_path.parent / "inject"), + ) + + +def _report_pid(_): + return os.getpid() + + +def _shards(tmp_path): + return sorted(p.name for p in tmp_path.glob("inject-log.json.*")) + + +def test_install_is_what_wires_the_fork_worker_finalizer(tmp_path): + """Go through ``install()``, not the private method. + + Calling ``_install_fork_worker_finalizer()`` directly passes even if the + single line wiring it into ``install()`` is deleted -- and that line sits in + the merge-conflict region with #287, so a conflict resolution could drop it + silently. This test is the only thing that would notice. + """ + if "fork" not in multiprocessing.get_all_start_methods(): + return + + log_path = tmp_path / "inject-log.json" + tracker = _installable_tracker(log_path) + + saved_open, saved_import = builtins.open, builtins.__import__ + try: + tracker.install() + process = multiprocessing.get_context("fork").Process( + target=_record_fork_only_import, + args=(tracker,), + ) + process.start() + process.join(timeout=10) + finally: + builtins.open, builtins.__import__ = saved_open, saved_import + + assert process.exitcode == 0 + assert (tmp_path / f"inject-log.json.{process.pid}").exists() + + +def test_pool_context_manager_workers_still_report(tmp_path): + """``with Pool(...)`` exits via ``terminate()``, which SIGTERMs the workers. + + SIGTERM's default disposition kills them outright, so neither the + multiprocessing finalizer nor atexit runs. This is the common idiom -- and + the ``num_proc`` case the finalizer's own docstring cites -- so it has to + report, not just the ``close()``/``join()`` shape. + + Asserted against the workers that actually ran a task, since those + demonstrably got through the after-fork hook. A worker forked and killed + while still bootstrapping may write nothing: the eager write makes this + best-effort, not a guarantee, and an exact shard count would be flaky. + """ + if "fork" not in multiprocessing.get_all_start_methods(): + return + + log_path = tmp_path / "inject-log.json" + tracker = _installable_tracker(log_path) + tracker._install_fork_worker_finalizer() + + context = multiprocessing.get_context("fork") + with context.Pool(2) as pool: + worker_pids = set(pool.map(_report_pid, range(8))) + + assert worker_pids, "no worker ran a task" + for pid in worker_pids: + assert (tmp_path / f"inject-log.json.{pid}").exists(), ( + f"worker {pid} ran a task but never reported; shards: {_shards(tmp_path)}" + ) + # The parent writes via atexit, which has not run yet. + assert f"inject-log.json.{os.getpid()}" not in _shards(tmp_path) + + +def test_write_log_writes_a_per_pid_shard_not_the_shared_file(tmp_path): + log_path = tmp_path / "inject-log.json" + _tracker(log_path).write_log() + assert not log_path.exists() # the shared path is NOT truncated + assert (tmp_path / f"inject-log.json.{os.getpid()}").exists() # the shard is + + +def test_real_fork_worker_writes_its_own_pid_shard(tmp_path): + """Linux multiprocessing fork workers use os._exit, so ordinary atexit does + not run. The multiprocessing finalizer must write the worker's real shard.""" + if "fork" not in multiprocessing.get_all_start_methods(): + return + + log_path = tmp_path / "inject-log.json" + tracker = _tracker(log_path) + tracker._install_fork_worker_finalizer() + process = multiprocessing.get_context("fork").Process( + target=_record_fork_only_import, + args=(tracker,), + ) + process.start() + process.join(timeout=10) + + assert process.exitcode == 0 + worker_shard = tmp_path / f"inject-log.json.{process.pid}" + assert worker_shard.exists() + payload = json.loads(worker_shard.read_text()) + assert payload["pid"] == process.pid + assert "fork_only_dependency" in payload["imported_modules"] + assert not (tmp_path / f"inject-log.json.{os.getpid()}").exists() + + +def test_a_forced_exit_keeps_the_fork_time_shard_but_loses_later_imports(tmp_path): + """Document the lifecycle boundary precisely. + + A worker that calls ``os._exit`` bypasses multiprocessing cleanup as well as + atexit, so no exit hook runs for it -- and the same is true of one killed by + SIGTERM or SIGKILL. The eager write at fork means such a worker still + contributes the state it inherited, rather than nothing at all. + + What is lost is what it imported *after* forking. That is the honest + remainder, and closing it needs incremental import journaling rather than an + exit hook. This test pins both halves so neither claim drifts. + """ + if "fork" not in multiprocessing.get_all_start_methods(): + return + + log_path = tmp_path / "inject-log.json" + tracker = _tracker(log_path) + tracker._install_fork_worker_finalizer() + process = multiprocessing.get_context("fork").Process( + target=_record_then_force_exit, + args=(tracker,), + ) + process.start() + process.join(timeout=10) + + assert process.exitcode == 0 + worker_shard = tmp_path / f"inject-log.json.{process.pid}" + assert worker_shard.exists(), "the fork-time snapshot should survive a forced exit" + payload = json.loads(worker_shard.read_text()) + assert "fork_only_dependency" not in payload["imported_modules"] + + +def test_worker_shard_does_not_clobber_the_workload_record(tmp_path): + """MMA's litdata case: same command, a worker shard with argv ['-c'] and a + subset of packages, plus the workload shard with the real command and the + full set. The merge must keep the workload identity and union the packages.""" + base = tmp_path / "inject-log.json" + # A litdata worker: sparse, argv ['-c'], few modules. + _shard( + base, + 222, + { + "argv": ["-c"], + "modules_files": ["/sp/multiprocessing/spawn.py"], + "used_packages": {"litdata": "0.2.59"}, + "imported_modules": ["litdata"], + "opened_files": ["/data/shard-0.bin"], + "installed_packages": {"litdata": "0.2.59"}, + "python_version": "3.12.10", + }, + ) + # The workload: the real command, the full package set (torch/lightning/...). + _shard( + base, + 111, + { + "argv": ["train.py", "--epochs", "3"], + "modules_files": [ + "/sp/torch/__init__.py", + "/sp/lightning/__init__.py", + "/repo/train.py", + ], + "used_packages": {"torch": "2.7.0", "lightning": "2.6.5", "litdata": "0.2.59"}, + "imported_modules": ["torch", "lightning", "litdata"], + "opened_files": ["/repo/train.py"], + "installed_packages": {"torch": "2.7.0", "lightning": "2.6.5", "litdata": "0.2.59"}, + "python_version": "3.12.10", + }, + ) + + merge_inject_logs(str(base)) + merged = json.loads(base.read_text(encoding="utf-8")) + + # Identity comes from the workload (richest shard), not the ['-c'] worker. + assert merged["argv"] == ["train.py", "--epochs", "3"] + # Packages/files/imports are the UNION across the tree. + assert merged["used_packages"] == {"torch": "2.7.0", "lightning": "2.6.5", "litdata": "0.2.59"} + assert set(merged["imported_modules"]) == {"torch", "lightning", "litdata"} + assert ( + "/repo/train.py" in merged["opened_files"] and "/data/shard-0.bin" in merged["opened_files"] + ) + # Shards are consumed. + assert not list(tmp_path.glob("inject-log.json.*")) + + +def test_merge_prefers_a_concrete_version_over_none(tmp_path): + base = tmp_path / "inject-log.json" + _shard(base, 1, {"modules_files": ["/sp/a.py"], "used_packages": {"wandb": None}}) + _shard( + base, 2, {"modules_files": ["/sp/a.py", "/sp/b.py"], "used_packages": {"wandb": "0.16.0"}} + ) + merge_inject_logs(str(base)) + assert json.loads(base.read_text())["used_packages"]["wandb"] == "0.16.0" + + +def test_merge_is_noop_without_shards(tmp_path): + base = tmp_path / "inject-log.json" + merge_inject_logs(str(base)) # no shards on disk + assert not base.exists() diff --git a/tests/execution/runtime/test_roar_footprint_location.py b/tests/execution/runtime/test_roar_footprint_location.py new file mode 100644 index 00000000..0f5d6163 --- /dev/null +++ b/tests/execution/runtime/test_roar_footprint_location.py @@ -0,0 +1,76 @@ +"""P0-11 (broad) / P0-28: roar's own dependency footprint must be subtracted from +the freeze by the LOCATION it loaded from, never by package NAME. + +The campaign runs roar in its own ``uv tool`` venv, ABI-matched to the workload, +so roar's deps and the workload's deps live in two different venvs but share +distribution *names* (e.g. ``typing_extensions`` is a dep of both roar's pydantic +and the workload's torch). Name-keyed subtraction (the reverted rc3 attempt) +stripped the workload's own copy — P0-28. Location-keyed subtraction removes only +what actually loaded from roar's install root, so the workload's copy survives. +""" + +from __future__ import annotations + +from roar.execution.runtime.inject.tracker import ( + _site_packages_top, + get_used_packages, + is_under_any_runtime_path, + roar_footprint_paths, +) + + +def test_footprint_excluded_by_location_not_by_name(tmp_path): + """The regression test for P0-28: a workload dependency that shares a NAME with + a roar dependency survives, because we key on where it loaded from.""" + roar_root = tmp_path / "uv-tools" / "roar-cli" / "lib" / "python3.12" / "site-packages" + wl_root = tmp_path / "wlvenv" / "lib" / "python3.12" / "site-packages" + inject_dir = str(roar_root / "roar" / "execution" / "runtime" / "inject") + wl_prefix = str(tmp_path / "wlvenv") + + # Loaded modules: roar's OWN click + typing_extensions (from roar_root), and the + # WORKLOAD's OWN typing_extensions + torch (from wl_root). typing_extensions + # collides on name across the two venvs — the exact P0-28 case. + loaded = [ + str(roar_root / "click" / "__init__.py"), + str(roar_root / "typing_extensions.py"), + str(wl_root / "typing_extensions.py"), + str(wl_root / "torch" / "__init__.py"), + ] + installed = {"click": "8.1.0", "typing_extensions": "4.16.0", "torch": "2.7.0"} + roar_dep_names = {"click", "typing_extensions"} # both are roar deps, by name + + # (1) NAME-keying (rc3) would drop the workload's typing_extensions as well — + # the false negative P0-28 reported. + name_kept = [f for f in loaded if _site_packages_top(f) not in roar_dep_names] + assert not any("typing_extensions" in f for f in name_kept), ( + "name-keying strips the workload's own typing_extensions — this is the P0-28 bug" + ) + + # (2) LOCATION-keying (the fix): exclude only what loaded from roar's root. + excl = roar_footprint_paths(inject_dir, wl_prefix) + assert excl, "isolated roar install root must be excluded" + loc_kept = [f for f in loaded if not is_under_any_runtime_path(f, excl)] + used = get_used_packages(loc_kept, installed) + + # (3) the workload's typing_extensions and torch survive; roar's click is gone. + assert "typing_extensions" in used, "workload's own typing_extensions must survive" + assert "torch" in used + assert "click" not in used, "roar's footprint must be gone" + + +def test_shared_venv_is_not_location_excluded(tmp_path): + """When roar is pip-installed in the workload's own venv, its root is under the + interpreter prefix; path cannot tell the copies apart, so no location exclusion + is applied and the freeze safely over-includes (never a false negative).""" + venv = tmp_path / "venv" + inject_dir = str( + venv / "lib" / "python3.12" / "site-packages" / "roar" / "execution" / "runtime" / "inject" + ) + assert roar_footprint_paths(inject_dir, str(venv)) == () + + +def test_isolated_root_is_reported(tmp_path): + """The isolated ``uv tool`` root (outside the workload prefix) is returned.""" + roar_root = tmp_path / "uv-tools" / "roar-cli" / "lib" / "python3.12" / "site-packages" + inject_dir = str(roar_root / "roar" / "execution" / "runtime" / "inject") + assert roar_footprint_paths(inject_dir, str(tmp_path / "wlvenv")) == (str(roar_root),) diff --git a/tests/execution/runtime/test_runtime_tracker.py b/tests/execution/runtime/test_runtime_tracker.py index 165d2c68..a2f897d7 100644 --- a/tests/execution/runtime/test_runtime_tracker.py +++ b/tests/execution/runtime/test_runtime_tracker.py @@ -2,9 +2,16 @@ import json import sys +from pathlib import Path + +import pytest from roar.execution.runtime.inject.tracker import ( RuntimeInjectionTracker, + get_active_runtime_pythonpath, + get_installed_packages, + get_used_packages, + merge_inject_logs, ) @@ -31,6 +38,7 @@ def handle_import(self, module_name: str, module) -> None: assert tracker.patched_environ_get("VIRTUAL_ENV") == "/tmp/venv" tracker.write_log() + merge_inject_logs(str(log_path)) # write_log writes a per-PID shard; merge -> canonical payload = json.loads(log_path.read_text(encoding="utf-8")) assert str(data_path.resolve()) in payload["opened_files"] assert payload["env_reads"]["VIRTUAL_ENV"] == "/tmp/venv" @@ -75,10 +83,43 @@ def handle_import(self, module_name: str, module) -> None: sys.path.remove(str(runtime_root)) sys.modules.pop("runtime_only", None) + merge_inject_logs(str(log_path)) # write_log writes a per-PID shard; merge -> canonical payload = json.loads(log_path.read_text(encoding="utf-8")) assert str(runtime_module) not in payload["modules_files"] +def test_package_version_comes_from_imported_workload_distribution( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + workload = tmp_path / "workload" / "site-packages" + runtime = tmp_path / "runtime" / "site-packages" + + def write_distribution(root: Path, version: str) -> Path: + root.mkdir(parents=True) + module = root / "shadowpkg.py" + module.write_text("VALUE = 1\n", encoding="utf-8") + metadata = root / f"shadowpkg-{version}.dist-info" + metadata.mkdir() + (metadata / "METADATA").write_text( + f"Metadata-Version: 2.1\nName: shadowpkg\nVersion: {version}\n", + encoding="utf-8", + ) + (metadata / "top_level.txt").write_text("shadowpkg\n", encoding="utf-8") + return module + + workload_module = write_distribution(workload, "1.0") + write_distribution(runtime, "9.0") + monkeypatch.setattr(sys, "path", [str(workload), str(runtime), *sys.path]) + runtime_paths = get_active_runtime_pythonpath({"ROAR_RUNTIME_PYTHONPATH_ACTIVE": str(runtime)}) + + installed = get_installed_packages(excluded_paths=runtime_paths) + used = get_used_packages([str(workload_module)], installed) + + assert installed["shadowpkg"] == "1.0" + assert used["shadowpkg"] == "1.0" + + def test_runtime_tracker_excludes_roar_internal_env_reads(tmp_path) -> None: """roar's own injected vars must not leak into captured env_reads (issue #164).""" log_path = tmp_path / "inject-log.json" @@ -107,6 +148,7 @@ def handle_import(self, module_name: str, module) -> None: assert tracker.patched_environ_get("HOME") == "/home/ubuntu" tracker.write_log() + merge_inject_logs(str(log_path)) # write_log writes a per-PID shard; merge -> canonical payload = json.loads(log_path.read_text(encoding="utf-8")) env_reads = payload["env_reads"] # User-facing reads are kept; roar's reserved namespace is dropped. diff --git a/tests/execution/runtime/test_sitecustomize_path_order.py b/tests/execution/runtime/test_sitecustomize_path_order.py index 11830572..96f08b5f 100644 --- a/tests/execution/runtime/test_sitecustomize_path_order.py +++ b/tests/execution/runtime/test_sitecustomize_path_order.py @@ -1,15 +1,19 @@ -"""sitecustomize prepends ROAR_RUNTIME_PYTHONPATH entries (in order). - -Behavior under test: when the traced Python doesn't already have roar -importable (the cross-Python lazy-install scenario), ``sitecustomize.py`` -must put the entries from ``ROAR_RUNTIME_PYTHONPATH`` at the *front* of -``sys.path``, preserving the declared order. Appending (the old behavior) -lets the system's stale site-packages win — which is the friction-journal -bug where lazy-installed ``typing_extensions`` 4.15.0 lost to the -system's 4.4.x. - -Tested via subprocess so we exercise the real sitecustomize module-import -side effects without polluting the test process's ``sys.path``. +"""sitecustomize places ROAR_RUNTIME_PYTHONPATH entries with the right precedence. + +When the traced Python can't already import roar (the cross-Python / +lazy-install scenario), ``sitecustomize.py`` adds safe +``ROAR_RUNTIME_PYTHONPATH`` entries to ``sys.path`` with two precedences: + +- a non-conflicting **ABI-matched runtime cache** is **prepended** so it can + beat the child's system copies; +- a cache with any workload import-name collision is not activated; +- everything else (roar's host site-packages, added only so a cross-interpreter + child can import roar) is **appended** — prepending it was P0-14: roar's host + packages shadowed the workload's recorded pins and the run executed against + host packages. + +Tested via subprocess so we exercise the real sitecustomize module-import side +effects without polluting the test process's ``sys.path``. """ from __future__ import annotations @@ -22,11 +26,21 @@ SOURCE_ROOT = Path(__file__).resolve().parents[3] +# Patches find_spec("roar") -> None so the add-path codepath runs, then imports +# sitecustomize. Callers append their own print statements. +_PATCH_AND_IMPORT = """ +import importlib.util, importlib, sys +_real = importlib.util.find_spec +importlib.util.find_spec = lambda name, *a, **k: None if name == "roar" else _real(name, *a, **k) +_sitecustomize = importlib.import_module("roar.execution.runtime.inject.sitecustomize") +""" + def _run_python( code: str, *, roar_runtime_pythonpath: str | None = None, + extra_env: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: """Run a subprocess Python with sitecustomize loaded from this source tree.""" env = dict(os.environ) @@ -38,6 +52,8 @@ def _run_python( else: env.pop("ROAR_RUNTIME_PYTHONPATH", None) env.pop("ROAR_WRAP", None) # skip the backend-dispatch gate; we only care about path order + if extra_env: + env.update(extra_env) return subprocess.run( [sys.executable, "-c", code], capture_output=True, @@ -49,61 +65,127 @@ def _run_python( ) -def test_runtime_pythonpath_entries_land_at_front_in_declared_order(tmp_path: Path) -> None: - """When roar isn't already importable, ROAR_RUNTIME_PYTHONPATH wins.""" - fake_runtime = tmp_path / "fake-runtime" - fake_runtime.mkdir() - fake_other = tmp_path / "fake-other" - fake_other.mkdir() +def test_abi_matched_cache_is_prepended(tmp_path: Path) -> None: + """roar's ABI-matched runtime cache must beat system copies -> prepended.""" + cache_home = tmp_path / "xdg" + cache_dir = cache_home / "roar" / "runtime" / "cp999" / "site-packages" + cache_dir.mkdir(parents=True) - # Force find_spec("roar") to return None by monkey-patching it before - # sitecustomize runs. We mark roar's site-packages location empty for the - # purposes of this subprocess by inserting a stub finder ahead of it that - # claims "roar is missing" — that's what triggers the prepend codepath. - code = textwrap.dedent( + code = _PATCH_AND_IMPORT + "print(f'first={sys.path[0]}')\n" + result = _run_python( + code, + roar_runtime_pythonpath=str(cache_dir), + extra_env={"XDG_CACHE_HOME": str(cache_home)}, + ) + assert result.returncode == 0, result.stderr + assert f"first={cache_dir}" in result.stdout, result.stdout + + +def test_abi_cache_with_workload_collision_is_not_activated(tmp_path: Path) -> None: + cache_home = tmp_path / "xdg" + cache_dir = cache_home / "roar" / "runtime" / "cp999" / "site-packages" + workload_dir = tmp_path / "workload" + cache_dir.mkdir(parents=True) + workload_dir.mkdir() + (cache_dir / "shadowpkg.py").write_text("MARK = 'cache'\n", encoding="utf-8") + (workload_dir / "shadowpkg.py").write_text("MARK = 'workload'\n", encoding="utf-8") + + code = _PATCH_AND_IMPORT + textwrap.dedent( + f""" + import os, shadowpkg + target = {str(cache_dir)!r} + print("cache_present=" + str(target in sys.path)) + print("winner=" + shadowpkg.MARK) + print("collisions=" + os.environ.get("ROAR_RUNTIME_CACHE_COLLISIONS", "")) """ - import importlib.util - import sys + ) + result = _run_python( + code, + roar_runtime_pythonpath=str(cache_dir), + extra_env={ + "XDG_CACHE_HOME": str(cache_home), + "PYTHONPATH": os.pathsep.join([str(SOURCE_ROOT), str(workload_dir)]), + }, + ) + assert result.returncode == 0, result.stderr + assert "cache_present=False" in result.stdout, result.stdout + assert "winner=workload" in result.stdout, result.stdout + assert "collisions=shadowpkg" in result.stdout, result.stdout - _real_find_spec = importlib.util.find_spec - def _patched_find_spec(name, *args, **kwargs): - if name == "roar": - return None - return _real_find_spec(name, *args, **kwargs) - importlib.util.find_spec = _patched_find_spec - import importlib - importlib.import_module("roar.execution.runtime.inject.sitecustomize") - # The prepend has run; assert the entries are at the front, in order. - print(f"first={sys.path[0]}") - print(f"second={sys.path[1]}") +def test_in_process_repair_degrades_instead_of_activating_a_colliding_cache( + tmp_path: Path, +) -> None: + cache_dir = tmp_path / "cache" / "site-packages" + workload_dir = tmp_path / "workload" + cache_dir.mkdir(parents=True) + workload_dir.mkdir() + (cache_dir / "shadowpkg.py").write_text("MARK = 'cache'\n", encoding="utf-8") + (workload_dir / "shadowpkg.py").write_text("MARK = 'workload'\n", encoding="utf-8") + + code = _PATCH_AND_IMPORT + textwrap.dedent( + f""" + import os + from pathlib import Path + from roar.execution.runtime import lazy_install + + lazy_install.ensure_runtime = lambda **_kwargs: Path({str(cache_dir)!r}) + repaired = _sitecustomize._repair_runtime_in_process("cpython-999") + print("repaired=" + str(repaired)) + print("cache_present=" + str({str(cache_dir)!r} in sys.path)) + print("collisions=" + os.environ.get("ROAR_RUNTIME_CACHE_COLLISIONS", "")) + print(_sitecustomize._runtime_gate_degrade_message((3, 99))) """ ) result = _run_python( code, - roar_runtime_pythonpath=os.pathsep.join([str(fake_runtime), str(fake_other)]), + extra_env={ + "PYTHONPATH": os.pathsep.join([str(SOURCE_ROOT), str(workload_dir)]), + }, + ) + assert result.returncode == 0, result.stderr + assert "repaired=False" in result.stdout, result.stdout + assert "cache_present=False" in result.stdout, result.stdout + assert "collisions=shadowpkg" in result.stdout, result.stdout + assert "Runtime cache disabled to preserve workload imports: shadowpkg." in result.stdout + + +def test_host_site_packages_are_appended_not_prepended(tmp_path: Path) -> None: + """P0-14: a non-cache runtime entry is appended, so it can't shadow the + workload — present on sys.path, but not at the front.""" + fake_host = tmp_path / "fake-host" + fake_host.mkdir() + + code = _PATCH_AND_IMPORT + textwrap.dedent( + f""" + target = {str(fake_host)!r} + print("present=" + str(target in sys.path)) + print("at_front=" + str(sys.path[0] == target)) + print("at_back=" + str(sys.path[-1] == target)) + """ ) + result = _run_python(code, roar_runtime_pythonpath=str(fake_host)) assert result.returncode == 0, result.stderr out = result.stdout - assert f"first={fake_runtime}" in out, out - assert f"second={fake_other}" in out, out + assert "present=True" in out, out + assert "at_front=False" in out, out + assert "at_back=True" in out, out -def test_no_prepend_when_roar_already_importable(tmp_path: Path) -> None: - """When roar is already importable, the function early-returns and leaves sys.path alone.""" +def test_no_change_when_roar_already_importable(tmp_path: Path) -> None: + """When roar is already importable, the function early-returns and leaves + sys.path alone.""" fake_runtime = tmp_path / "fake-runtime" fake_runtime.mkdir() code = textwrap.dedent( f""" - import importlib - import sys - # Roar IS importable (PYTHONPATH points at source root). Prepend should no-op. + import importlib, sys + # Roar IS importable (PYTHONPATH points at source root). Add-path should no-op. importlib.import_module("roar.execution.runtime.inject.sitecustomize") target = {str(fake_runtime)!r} - in_top_three = target in sys.path[:3] - print("in_top_three=" + str(in_top_three)) + print("present=" + str(target in sys.path)) """ ) result = _run_python(code, roar_runtime_pythonpath=str(fake_runtime)) assert result.returncode == 0, result.stderr - assert "in_top_three=False" in result.stdout, result.stdout + assert "present=False" in result.stdout, result.stdout diff --git a/tests/execution/runtime/test_used_packages_by_name.py b/tests/execution/runtime/test_used_packages_by_name.py new file mode 100644 index 00000000..990c0d21 --- /dev/null +++ b/tests/execution/runtime/test_used_packages_by_name.py @@ -0,0 +1,208 @@ +"""P0-6 / P0-13: the name pass recovers a genuinely-used package the file pass +mis-attributed because the import was ALIASED (e.g. `sys.modules["wandb"] = +trackio`) — and ONLY that case. It must not attribute a normally-loaded import +(file pass's job) nor a merely-probed optional import that happens to be +installed (P0-13: `accelerate` probing `sagemaker` on a SageMaker AMI). + +Aliasing is detected via `loaded_files` (name -> the module file actually loaded +for it): a name is attributed only when its loaded module lives in a *different* +site-packages package than the name. +""" + +from __future__ import annotations + +import importlib.metadata as ilm +import json +import os +import sys +import types + +from roar.execution.runtime.inject.tracker import ( + RuntimeInjectionTracker, + get_used_packages, +) + + +def _loaded_as(pkg: str) -> str: + """A site-packages module file for top-level package ``pkg``.""" + return f"/venv/lib/python3.12/site-packages/{pkg}/__init__.py" + + +def test_aliased_import_attributed_by_name(monkeypatch): + """`import wandb` aliased to trackio: file pass sees only trackio, but the + name pass sees wandb was imported yet loaded a *different* package -> records + wandb.""" + monkeypatch.setattr(ilm, "packages_distributions", lambda: {"wandb": ["wandb"]}) + used = get_used_packages( + modules_files=[], + installed_packages={"wandb": "0.16.0", "trackio": "0.1.0"}, + imported_modules=["wandb"], + loaded_files={"wandb": _loaded_as("trackio")}, # aliased + ) + assert used == {"wandb": "0.16.0"} + + +def test_probed_optional_import_is_not_attributed(monkeypatch): + """P0-13 (#264 regression): an optional import that merely happened to be + installed (loaded as ITSELF, or not loaded at all) is not aliased, so the + name pass leaves it out — no unsatisfiable substrate in the freeze.""" + monkeypatch.setattr(ilm, "packages_distributions", lambda: {"sagemaker": ["sagemaker-core"]}) + installed = {"sagemaker-core": "2.9.0"} + # loaded as itself (a real but merely-probed import) -> file pass's job, not ours + used_self = get_used_packages( + modules_files=[], + installed_packages=installed, + imported_modules=["sagemaker"], + loaded_files={"sagemaker": _loaded_as("sagemaker")}, + ) + # probed via find_spec / lazy import -> never in loaded_files at all + used_absent = get_used_packages( + modules_files=[], + installed_packages=installed, + imported_modules=["sagemaker"], + loaded_files={}, + ) + assert "sagemaker-core" not in used_self + assert "sagemaker-core" not in used_absent + + +def test_never_imported_package_is_not_added(monkeypatch): + monkeypatch.setattr(ilm, "packages_distributions", lambda: {"wandb": ["wandb"]}) + used = get_used_packages( + modules_files=[], + installed_packages={"wandb": "0.16.0"}, + imported_modules=["numpy", "os"], # wandb never imported + loaded_files={"wandb": _loaded_as("trackio")}, # even if (somehow) aliased + ) + assert "wandb" not in used + + +def test_imported_but_not_installed_and_tracer_never_attributed(monkeypatch): + """An aliased name that isn't installed is skipped; roar is never attributed.""" + monkeypatch.setattr( + ilm, "packages_distributions", lambda: {"ghost": ["ghost"], "roar": ["roar-cli"]} + ) + used = get_used_packages( + modules_files=[], + installed_packages={"roar-cli": "0.4.4"}, # 'ghost' not installed + imported_modules=["ghost", "roar", "roar.execution.runtime"], + loaded_files={"ghost": _loaded_as("ghost_alias"), "roar": _loaded_as("roar_rt")}, + ) + assert used == {} + + +def test_shadowed_import_recorded_through_write_log(tmp_path, monkeypatch): + """End-to-end through the real capture path: tracking_import records the name, + write_log builds loaded_files from sys.modules and runs get_used_packages, and + the aliased package lands in the log's used_packages.""" + from roar.execution.runtime.inject import tracker as tmod + + log_path = tmp_path / "log.json" + + class _Ctl: + def handle_import(self, *args, **kwargs): + return None + + tracker = RuntimeInjectionTracker( + {"ROAR_LOG_FILE": str(log_path)}, + _Ctl(), + log_file=str(log_path), + inject_dir=str(tmp_path / "inject"), + ) + # get_installed_packages now takes excluded_paths (#268); accept and ignore it. + monkeypatch.setattr(tmod, "get_installed_packages", lambda **_: {"wandb": "0.16.0"}) + monkeypatch.setattr(ilm, "packages_distributions", lambda: {"wandb": ["wandb"]}) + + # The shim: `wandb` resolves to a stand-in whose __file__ is trackio's, so the + # file pass records trackio, not wandb — but the name pass detects the alias. + stand_in = types.ModuleType("trackio_standin") + stand_in.__file__ = _loaded_as("trackio") + monkeypatch.setitem(sys.modules, "wandb", stand_in) + tracker.tracking_import("wandb") + + tracker.write_log() + shard = log_path.with_name(f"{log_path.name}.{os.getpid()}") + written = log_path if log_path.exists() else shard # canonical, or per-PID shard (P0-9) + payload = json.loads(written.read_text()) + assert "wandb" in payload["imported_modules"] + assert payload["used_packages"].get("wandb") == "0.16.0" + + +def test_roar_is_never_recorded_in_the_freeze_via_file_pass(monkeypatch): + """P0-11: the file pass must skip roar (roar-cli is installed separately and + unpinned; a dev build would otherwise pin an unresolvable roar-cli==X.Y.dev0).""" + monkeypatch.setattr(ilm, "packages_distributions", lambda: {"roar": ["roar-cli"]}) + used = get_used_packages( + modules_files=["/venv/lib/python3.12/site-packages/roar/__init__.py"], + installed_packages={"roar-cli": "0.4.4.dev0"}, + ) + assert "roar-cli" not in used and "roar" not in used + + +class _FakeDist: + def __init__(self, path): + self._path = path + + def locate_file(self, rel=""): + return self._path + + +def test_self_package_from_editable_is_not_pinned(tmp_path, monkeypatch): + """P0-12: the workload's own `pip install -e .` package loads from the repo, + not site-packages, so it isn't aliased and the name pass leaves it out.""" + monkeypatch.setattr(ilm, "packages_distributions", lambda: {"mypkg": ["mypkg"]}) + used = get_used_packages( + modules_files=[], + installed_packages={"mypkg": "1.0"}, + imported_modules=["mypkg"], + loaded_files={"mypkg": str(tmp_path / "repo" / "mypkg" / "__init__.py")}, + workload_root=str(tmp_path / "repo"), + ) + assert "mypkg" not in used + + +def test_real_aliased_dep_outside_repo_is_still_pinned(tmp_path, monkeypatch): + """A genuinely aliased dep whose metadata is outside the repo is still + recorded (the P0-12 repo skip must not drop it).""" + repo = tmp_path / "repo" + repo.mkdir() + dist_info = tmp_path / "site-packages" / "wandb-0.16.0.dist-info" + dist_info.mkdir(parents=True) + monkeypatch.setattr(ilm, "packages_distributions", lambda: {"wandb": ["wandb"]}) + monkeypatch.setattr(ilm, "distribution", lambda name: _FakeDist(dist_info)) + used = get_used_packages( + modules_files=[], + installed_packages={"wandb": "0.16.0"}, + imported_modules=["wandb"], + loaded_files={"wandb": _loaded_as("trackio")}, # aliased + workload_root=str(repo), + ) + assert used.get("wandb") == "0.16.0" + + +def test_dist_packages_import_is_recorded(monkeypatch): + """P0-18: a package loaded from dist-packages (system Python, e.g. the cert + AMI) must reach the freeze — the file pass previously only matched + site-packages, so it was dropped entirely.""" + monkeypatch.setattr( + ilm, "packages_distributions", lambda: {"huggingface_hub": ["huggingface-hub"]} + ) + used = get_used_packages( + modules_files=["/usr/lib/python3/dist-packages/huggingface_hub/__init__.py"], + installed_packages={"huggingface-hub": "1.27.0"}, + ) + assert used.get("huggingface-hub") == "1.27.0" + + +def test_failed_probe_import_is_not_recorded_even_with_dist_packages(monkeypatch): + """P0-13 must stay fixed: a probed import that FAILED (never loaded, so not in + modules_files/loaded_files — only its name is in imported_modules) is not + recorded, even now that dist-packages is recognized.""" + monkeypatch.setattr(ilm, "packages_distributions", lambda: {"sagemaker": ["sagemaker-core"]}) + used = get_used_packages( + modules_files=[], # import sagemaker failed -> not loaded + installed_packages={"sagemaker-core": "2.9.0"}, + imported_modules=["sagemaker"], # name recorded before the failed import + loaded_files={}, # not in sys.modules + ) + assert "sagemaker-core" not in used diff --git a/tests/integration/fake_glaas.py b/tests/integration/fake_glaas.py index 4e7dd90c..9d1bfff5 100644 --- a/tests/integration/fake_glaas.py +++ b/tests/integration/fake_glaas.py @@ -22,6 +22,7 @@ def __init__(self) -> None: self.registration_session_job_batches: list[dict[str, Any]] = [] self.registration_session_job_creates: list[dict[str, Any]] = [] self.artifact_batches: list[list[dict[str, Any]]] = [] + self.registration_session_artifact_batches: list[list[dict[str, Any]]] = [] self.auth_headers: list[dict[str, Any]] = [] self.input_links: list[dict[str, Any]] = [] self.output_links: list[dict[str, Any]] = [] @@ -34,6 +35,8 @@ def __init__(self) -> None: self.current_labels_by_target: dict[str, dict[str, Any]] = {} self.label_history_by_target: dict[str, list[dict[str, Any]]] = {} self.composite_registrations: list[dict[str, Any]] = [] + self.registration_session_composite_registrations: list[dict[str, Any]] = [] + self.registration_session_view_edges: list[dict[str, Any]] = [] self.artifacts_by_digest: dict[str, dict[str, Any]] = {} self.artifact_dags_by_digest: dict[str, dict[str, Any]] = {} self.session_reproductions_by_hash: dict[str, dict[str, Any]] = {} @@ -160,6 +163,32 @@ def _record_artifacts(self, artifacts: list[dict[str, Any]]) -> None: if isinstance(digest, str) and digest: self.server.artifacts_by_digest[digest] = artifact + def _record_label_sync(self, labels: list[dict[str, Any]]) -> None: + self.server.label_syncs.append(labels) + for label in labels: + target_key = _label_target_key(label) + current = self.server.current_labels_by_target.get(target_key) + version = int(current.get("version", 0)) + 1 if isinstance(current, dict) else 1 + current_label = { + "id": f"label-{len(self.server.current_labels_by_target) + 1}", + "entityType": label.get("entity_type"), + "version": version, + "metadata": label.get("metadata") + if isinstance(label.get("metadata"), dict) + else {}, + "createdAt": "2026-01-01T00:00:00Z", + } + if label.get("entity_type") == "dag": + current_label["sessionHash"] = label.get("session_hash") + elif label.get("entity_type") == "job": + current_label["sessionHash"] = label.get("session_hash") + current_label["jobUid"] = label.get("job_uid") + elif label.get("entity_type") == "artifact": + current_label["sessionHash"] = label.get("session_hash") + current_label["artifactHash"] = label.get("artifact_hash") + self.server.current_labels_by_target[target_key] = current_label + self.server.label_history_by_target.setdefault(target_key, []).append(current_label) + def _resolve_creator_identity(self, authenticated_user: dict[str, str] | None) -> str: if not isinstance(authenticated_user, dict): return "anonymous" @@ -202,8 +231,8 @@ def _count_registration_session_staging( jobs = [job for job in jobs_by_uid.values() if isinstance(job, dict)] return { "jobs": len(jobs), - "inputs": sum(len(job.get("inputs", [])) for job in jobs), - "outputs": sum(len(job.get("outputs", [])) for job in jobs), + "inputs": sum(len(_dedup_artifacts_by_hash(job.get("inputs", []))) for job in jobs), + "outputs": sum(len(_dedup_artifacts_by_hash(job.get("outputs", []))) for job in jobs), } def _compute_registration_session_hash( @@ -230,8 +259,7 @@ def _compute_registration_session_hash( inputs = sorted( [ {"hash": artifact.get("hash"), "path": artifact.get("path")} - for artifact in job.get("inputs", []) - if isinstance(artifact, dict) + for artifact in _dedup_artifacts_by_hash(job.get("inputs", [])) ], key=lambda artifact: ( str(artifact.get("hash") or ""), @@ -241,8 +269,7 @@ def _compute_registration_session_hash( outputs = sorted( [ {"hash": artifact.get("hash"), "path": artifact.get("path")} - for artifact in job.get("outputs", []) - if isinstance(artifact, dict) + for artifact in _dedup_artifacts_by_hash(job.get("outputs", [])) ], key=lambda artifact: ( str(artifact.get("hash") or ""), @@ -622,40 +649,104 @@ def do_POST(self) -> None: self._write_json(200, {"created": len(artifacts), "existing": 0}) return + registration_artifact_match = re.fullmatch( + r"/api/v1/registration-sessions/([^/]+)/artifacts/batch", + self.path, + ) + if registration_artifact_match: + registration_session_id = registration_artifact_match.group(1) + _authenticated_user, session_state = self._authorize_registration_session_write( + registration_session_id, + authorization, + ) + if session_state is None or session_state.get("status") != "active": + self._write_json(401, {"error": "Missing, invalid, or closed session"}) + return + artifacts = payload.get("artifacts", []) + if isinstance(artifacts, list): + self.server.registration_session_artifact_batches.append(artifacts) + self._record_artifacts(artifacts) + self._write_json(200, {"created": len(artifacts), "existing": 0}) + return + + registration_composite_match = re.fullmatch( + r"/api/v1/registration-sessions/([^/]+)/artifacts/composites", + self.path, + ) + if registration_composite_match: + registration_session_id = registration_composite_match.group(1) + _authenticated_user, session_state = self._authorize_registration_session_write( + registration_session_id, + authorization, + ) + if session_state is None or session_state.get("status") != "active": + self._write_json(401, {"error": "Missing, invalid, or closed session"}) + return + self.server.registration_session_composite_registrations.append(payload) + self._record_artifacts([payload]) + self._write_json( + 200, + { + "artifact_id": "registration-composite-" + f"{len(self.server.registration_session_composite_registrations)}", + "created": True, + }, + ) + return + if self.path == "/api/v1/labels/sync": labels = payload.get("labels", []) if isinstance(labels, list): - self.server.label_syncs.append(labels) - for label in labels: - if not isinstance(label, dict): - continue - target_key = _label_target_key(label) - current = self.server.current_labels_by_target.get(target_key) - version = int(current.get("version", 0)) + 1 if isinstance(current, dict) else 1 - current_label = { - "id": f"label-{len(self.server.current_labels_by_target) + 1}", - "entityType": label.get("entity_type"), - "version": version, - "metadata": label.get("metadata") - if isinstance(label.get("metadata"), dict) - else {}, - "createdAt": "2026-01-01T00:00:00Z", - } - if label.get("entity_type") == "dag": - current_label["sessionHash"] = label.get("session_hash") - elif label.get("entity_type") == "job": - current_label["sessionHash"] = label.get("session_hash") - current_label["jobUid"] = label.get("job_uid") - elif label.get("entity_type") == "artifact": - current_label["sessionHash"] = label.get("session_hash") - current_label["artifactHash"] = label.get("artifact_hash") - self.server.current_labels_by_target[target_key] = current_label - self.server.label_history_by_target.setdefault(target_key, []).append(current_label) + self._record_label_sync([label for label in labels if isinstance(label, dict)]) self._write_json( 200, {"created": 0, "updated": 0, "unchanged": len(labels)}, ) + return + + registration_label_match = re.fullmatch( + r"/api/v1/registration-sessions/([^/]+)/labels/batch", + self.path, + ) + if registration_label_match: + registration_session_id = registration_label_match.group(1) + _authenticated_user, session_state = self._authorize_registration_session_write( + registration_session_id, + authorization, + ) + if session_state is None: + self._write_json(401, {"error": "Missing or invalid auth"}) return + lineage_hash = session_state.get("hash") + if session_state.get("status") != "closed" or not isinstance(lineage_hash, str): + self._write_json( + 400, + {"error": {"message": "Registration session must be finalized"}}, + ) + return + raw_labels = payload.get("labels", []) + labels = ( + [ + {**label, "session_hash": lineage_hash} + for label in raw_labels + if isinstance(label, dict) + ] + if isinstance(raw_labels, list) + else [] + ) + self._record_label_sync(labels) + self._write_json( + 200, + { + "registration_session_id": registration_session_id, + "hash": lineage_hash, + "processed": len(labels), + "created": len(labels), + "updated": 0, + "noops": 0, + }, + ) + return if self.path == "/api/v1/labels/reconcile": if authenticated_user is None: @@ -1154,6 +1245,22 @@ def log_message(self, format: str, *args: object) -> None: """Suppress default stderr logging for integration tests.""" +def _dedup_artifacts_by_hash(artifacts: Any) -> list[dict[str, Any]]: + """Model glaas's ``(job_id, artifact_hash)`` storage key: byte-identical edges + written to several paths collapse to one stored row, keeping the + lexicographically-smallest path. Mirrors roar's staged-count / canonical dedup + so this fake counts and hashes the way the real server stores (P0-22).""" + by_hash: dict[str, dict[str, Any]] = {} + for artifact in artifacts if isinstance(artifacts, list) else []: + if not isinstance(artifact, dict) or not artifact.get("hash"): + continue + digest = artifact["hash"] + current = by_hash.get(digest) + if current is None or str(artifact.get("path") or "") < str(current.get("path") or ""): + by_hash[digest] = artifact + return list(by_hash.values()) + + def _parse_metadata_object(value: Any) -> dict[str, Any]: if isinstance(value, dict): return value @@ -1278,6 +1385,10 @@ def registration_session_job_creates(self) -> list[dict[str, Any]]: def artifact_batches(self) -> list[list[dict[str, Any]]]: return self._server.artifact_batches + @property + def registration_session_artifact_batches(self) -> list[list[dict[str, Any]]]: + return self._server.registration_session_artifact_batches + @property def auth_headers(self) -> list[dict[str, Any]]: return self._server.auth_headers diff --git a/tests/integration/test_cross_python_runtime_repair.py b/tests/integration/test_cross_python_runtime_repair.py index addce649..cac4da8e 100644 --- a/tests/integration/test_cross_python_runtime_repair.py +++ b/tests/integration/test_cross_python_runtime_repair.py @@ -32,8 +32,9 @@ _INJECT_DIR = str(Path(roar.__file__).resolve().parent / "execution" / "runtime" / "inject") # Roar's importable root + the site-packages carrying the *current* (and so, -# for a different-ABI worker, wrong-ABI) pydantic_core — mirrors what the tracer -# puts on ROAR_RUNTIME_PYTHONPATH for a cross-Python child. +# for a different-ABI worker, wrong-ABI) pydantic_core. The tracer owns these +# roots and declares them on ROAR_RUNTIME_PYTHONPATH; the workload's PYTHONPATH +# contains only the startup hook. _SOURCE_ROOT = str(Path(roar.__file__).resolve().parent.parent) _CURRENT_SITE_PACKAGES = str(Path(pydantic_core.__file__).resolve().parent.parent) @@ -103,7 +104,8 @@ def test_wrapper_launch_repairs_runtime_in_process_and_installs_once(tmp_path: P env = { **os.environ, - "PYTHONPATH": os.pathsep.join([_INJECT_DIR, _SOURCE_ROOT, _CURRENT_SITE_PACKAGES]), + "PYTHONPATH": _INJECT_DIR, + "ROAR_RUNTIME_PYTHONPATH": os.pathsep.join([_SOURCE_ROOT, _CURRENT_SITE_PACKAGES]), "ROAR_WRAP": "1", "XDG_CACHE_HOME": str(tmp_path / "xdg"), "PAYLOAD": _WORKER_PAYLOAD, diff --git a/tests/integration/test_no_crossenv_syspath_shadow.py b/tests/integration/test_no_crossenv_syspath_shadow.py new file mode 100644 index 00000000..2f1352bb --- /dev/null +++ b/tests/integration/test_no_crossenv_syspath_shadow.py @@ -0,0 +1,187 @@ +"""P0-14: roar's runtime injection must not let roar's own environment shadow the +workload's recorded packages on ``sys.path`` or in captured provenance. + +roar makes itself importable in a traced child by putting entries on +``ROAR_RUNTIME_PYTHONPATH``; ``sitecustomize`` applies them. Previously *all* of +them were prepended, so on a cross-interpreter run roar's host ``dist-packages`` +landed at ``sys.path[0]`` and shadowed the recorded pins (the run executed +against host packages, and could certify GREEN for the wrong reason). The fix: +append roar's host environment, and activate the ABI-matched runtime **cache** +only when none of its import names collide with the workload. + +These run a roar-less child interpreter with a ``sitecustomize`` on +``PYTHONPATH`` and assert which copy of a shadowed module wins. +""" + +from __future__ import annotations + +import json +import os +import platform +import sqlite3 +import subprocess +import sys +import venv +from collections.abc import Callable +from pathlib import Path + +import pytest + +import tests.conftest as test_conftest + +INJECT_DIR = Path(__file__).resolve().parents[2] / "roar" / "execution" / "runtime" / "inject" +SOURCE_ROOT = Path(__file__).resolve().parents[2] + + +def _roarless_python(tmp_path: Path) -> Path: + """A Python that cannot already import roar (so the injection path runs).""" + child = tmp_path / "child" + venv.EnvBuilder(with_pip=False).create(str(child)) + return child / ("Scripts" if sys.platform == "win32" else "bin") / "python" + + +def _write_pkg(root: Path, mark: str) -> None: + root.mkdir(parents=True, exist_ok=True) + (root / "shadowpkg.py").write_text(f"MARK = {mark!r}\n", encoding="utf-8") + + +def _write_distribution(root: Path, version: str) -> None: + _write_pkg(root, version) + metadata = root / f"shadowpkg-{version}.dist-info" + metadata.mkdir() + (metadata / "METADATA").write_text( + f"Metadata-Version: 2.1\nName: shadowpkg\nVersion: {version}\n", + encoding="utf-8", + ) + (metadata / "top_level.txt").write_text("shadowpkg\n", encoding="utf-8") + + +def _site_packages(python: Path) -> Path: + result = subprocess.run( + [str(python), "-c", "import sysconfig; print(sysconfig.get_paths()['purelib'])"], + capture_output=True, + text=True, + check=True, + ) + return Path(result.stdout.strip()) + + +def _run(child_py: Path, env: dict[str, str]) -> str: + r = subprocess.run( + [str(child_py), "-c", "import shadowpkg; print('WINNER=' + shadowpkg.MARK)"], + env=env, + capture_output=True, + text=True, + ) + for line in r.stdout.splitlines(): + if line.startswith("WINNER="): + return line[len("WINNER=") :] + raise AssertionError(f"no winner line.\nstdout={r.stdout!r}\nstderr={r.stderr!r}") + + +def test_host_site_packages_do_not_shadow_the_workload(tmp_path): + """A non-cache runtime entry (roar's host site-packages) is APPENDED, so the + workload's own copy (on PYTHONPATH) wins. Before the fix it was prepended and + 'host' won — a silent execution against host packages.""" + child_py = _roarless_python(tmp_path) + _write_pkg(tmp_path / "workload", "workload") + _write_pkg(tmp_path / "host", "host") + + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join([str(INJECT_DIR), str(tmp_path / "workload")]) + env["ROAR_RUNTIME_PYTHONPATH"] = str(tmp_path / "host") + env.pop("ROAR_WRAP", None) + + assert _run(child_py, env) == "workload" + + +def test_abi_matched_cache_does_not_shadow_the_workload(tmp_path): + """A cache entry that overlaps the workload is not activated.""" + child_py = _roarless_python(tmp_path) + cache_home = tmp_path / "xdg" + cache_pkg = cache_home / "roar" / "runtime" / "cp999" / "site-packages" + _write_pkg(cache_pkg, "cache") + _write_pkg(tmp_path / "workload", "workload") + + env = dict(os.environ) + env["XDG_CACHE_HOME"] = str(cache_home) + env["PYTHONPATH"] = os.pathsep.join([str(INJECT_DIR), str(tmp_path / "workload")]) + env["ROAR_RUNTIME_PYTHONPATH"] = str(cache_pkg) + env.pop("ROAR_WRAP", None) + + assert _run(child_py, env) == "workload" + + +@pytest.mark.skipif(platform.system() != "Linux", reason="product path uses Linux tracer") +def test_roar_run_records_the_distribution_that_the_workload_imported( + temp_git_repo: Path, + git_commit: Callable[[str], None], +) -> None: + """The child imports and records its own pin even when Roar's host has another. + + The workload is intentionally Roar-unaware. A separate parent venv imports this + worktree through a .pth file, while a child venv owns the package under test. + """ + test_conftest._ensure_repo_local_ptrace_tracer() + env_root = temp_git_repo.parent / f"{temp_git_repo.name}-crossenv" + parent_root = env_root / "parent" + child_root = env_root / "child" + venv.EnvBuilder(with_pip=False).create(str(parent_root)) + venv.EnvBuilder(with_pip=False).create(str(child_root)) + scripts_dir = "Scripts" if sys.platform == "win32" else "bin" + parent_python = parent_root / scripts_dir / "python" + child_python = child_root / scripts_dir / "python" + parent_site = _site_packages(parent_python) + child_site = _site_packages(child_python) + current_site = _site_packages(Path(sys.executable)) + + (parent_site / "roar-worktree.pth").write_text( + f"{SOURCE_ROOT}\n{current_site}\n", + encoding="utf-8", + ) + _write_distribution(parent_site, "9.0") + _write_distribution(child_site, "1.0") + + script = temp_git_repo / "workload.py" + script.write_text( + "import json, shadowpkg\n" + "with open('observed.json', 'w', encoding='utf-8') as handle:\n" + " json.dump({'version': shadowpkg.MARK, 'file': shadowpkg.__file__}, handle)\n", + encoding="utf-8", + ) + git_commit("add cross-environment workload") + + env = dict(os.environ) + env.pop("PYTHONPATH", None) + result = subprocess.run( + [ + str(parent_python), + "-m", + "roar", + "run", + "--tracer", + "ptrace", + "--no-tracer-fallback", + str(child_python), + script.name, + ], + cwd=temp_git_repo, + env=env, + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, f"stdout={result.stdout!r}\nstderr={result.stderr!r}" + + observed = json.loads((temp_git_repo / "observed.json").read_text(encoding="utf-8")) + assert observed["version"] == "1.0" + assert Path(observed["file"]).is_relative_to(child_site) + + connection = sqlite3.connect(temp_git_repo / ".roar" / "roar.db") + try: + row = connection.execute("SELECT metadata FROM jobs ORDER BY id DESC LIMIT 1").fetchone() + finally: + connection.close() + assert row is not None and row[0] + metadata = json.loads(row[0]) + assert metadata["packages"]["pip"]["shadowpkg"] == "1.0" diff --git a/tests/integration/test_put_cli_integration.py b/tests/integration/test_put_cli_integration.py index 3f5dabc7..c3865624 100644 --- a/tests/integration/test_put_cli_integration.py +++ b/tests/integration/test_put_cli_integration.py @@ -142,6 +142,7 @@ def test_put_registers_lineage_with_fake_glaas_and_updates_local_dag( assert len(fake_glaas_publish_server.job_batches) == 0 assert len(fake_glaas_publish_server.job_creates) == 0 assert len(fake_glaas_publish_server.artifact_batches) == 0 + assert len(fake_glaas_publish_server.registration_session_artifact_batches) == 1 assert len(fake_glaas_publish_server.registration_session_job_batches) == 1 assert len(fake_glaas_publish_server.registration_session_job_creates) == 1 assert fake_glaas_publish_server.registration_session_input_links diff --git a/tests/integration/test_python_capture_fail_closed.py b/tests/integration/test_python_capture_fail_closed.py new file mode 100644 index 00000000..e555eee3 --- /dev/null +++ b/tests/integration/test_python_capture_fail_closed.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import json +import shutil +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.integration + +_MACOS_PROTECTED_BINARY = pytest.mark.skipif( + sys.platform == "darwin", + reason="macOS protected system binaries reject preload before the workload starts", +) + +# `true` is /bin/true on most Linux distributions but only /usr/bin/true on +# macOS, where a hardcoded /bin/true exits 127 and looks like a roar failure. +# On macOS it is SIP-protected either way, so its test carries the skip above. +_TRUE_BINARY = shutil.which("true") or "/usr/bin/true" + + +def _roar(cwd: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-m", "roar", *args], + cwd=cwd, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + +def _latest_metadata(cwd: Path) -> dict: + connection = sqlite3.connect(cwd / ".roar" / "roar.db") + row = connection.execute("SELECT metadata FROM jobs ORDER BY id DESC LIMIT 1").fetchone() + connection.close() + assert row is not None + return json.loads(row[0]) + + +@pytest.mark.parametrize( + "command", + [ + pytest.param( + ["env", "PYTHONPATH=.", sys.executable, "-c", "import click"], + marks=_MACOS_PROTECTED_BINARY, + id="env-replace", + ), + pytest.param( + ["env", "-u", "PYTHONPATH", sys.executable, "-c", "import click"], + marks=_MACOS_PROTECTED_BINARY, + id="env-unset", + ), + pytest.param([sys.executable, "-E", "-c", "import click"], id="python-E"), + pytest.param([sys.executable, "-I", "-c", "import click"], id="python-I"), + pytest.param([sys.executable, "-S", "-c", "pass"], id="python-S"), + pytest.param( + ["sh", "-c", f"PYTHONPATH=. {sys.executable} -c 'import click'"], + marks=_MACOS_PROTECTED_BINARY, + id="shell-replace", + ), + ], +) +def test_suppressed_injection_warns_and_records_failed_capture( + tmp_path: Path, command: list[str] +) -> None: + assert _roar(tmp_path, "init", "-n").returncode == 0 + assert _roar(tmp_path, "tracer", "use", "preload").returncode == 0 + + run = _roar(tmp_path, "run", *command) + + assert run.returncode == 0 + assert "Python package capture did not complete" in run.stderr + assert _latest_metadata(tmp_path)["python_capture"] == "missing" + + +def test_successful_python_capture_has_no_warning(tmp_path: Path) -> None: + assert _roar(tmp_path, "init", "-n").returncode == 0 + assert _roar(tmp_path, "tracer", "use", "preload").returncode == 0 + + run = _roar(tmp_path, "run", sys.executable, "-c", "import click") + + assert run.returncode == 0 + assert "Python package capture did not complete" not in run.stderr + assert _latest_metadata(tmp_path)["python_capture"] == "complete" + + +@_MACOS_PROTECTED_BINARY +def test_non_python_command_is_not_misreported(tmp_path: Path) -> None: + assert _roar(tmp_path, "init", "-n").returncode == 0 + assert _roar(tmp_path, "tracer", "use", "preload").returncode == 0 + + run = _roar(tmp_path, "run", _TRUE_BINARY) + + assert run.returncode == 0 + assert "Python package capture did not complete" not in run.stderr + assert _latest_metadata(tmp_path)["python_capture"] == "not-applicable" diff --git a/tests/integration/test_recorded_command_is_what_was_run.py b/tests/integration/test_recorded_command_is_what_was_run.py new file mode 100644 index 00000000..b455bcdf --- /dev/null +++ b/tests/integration/test_recorded_command_is_what_was_run.py @@ -0,0 +1,100 @@ +"""Tests that a run records the command the user asked for. + +The root process's argv used to be read back from /proc, which reports what the +kernel ran rather than what was requested. A `#!/usr/bin/env python3` script +therefore recorded as `/usr/bin/env python3 ./train.sh`, and a process that +exited before the read recorded nothing at all -- so the same run could be +recorded two different ways depending on machine load. + +roar launches the workload, so its argv is known exactly. Descendants have no +such source and still come from /proc. +""" + +from __future__ import annotations + +import json +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.integration + +# On Apple Silicon the system launchers are arm64e platform binaries, and dyld +# refuses to insert the arm64 preload dylib into them: +# incompatible architecture (have 'arm64', need 'arm64e') +_MACOS_PROTECTED_BINARY = pytest.mark.skipif( + sys.platform == "darwin", + reason="macOS protected system binaries reject preload injection", +) + + +def _roar(cwd: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-m", "roar", *args], + cwd=cwd, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + + +def _latest_command(cwd: Path) -> list[str]: + connection = sqlite3.connect(cwd / ".roar" / "roar.db") + row = connection.execute("SELECT metadata FROM jobs ORDER BY id DESC LIMIT 1").fetchone() + connection.close() + assert row is not None + return (json.loads(row[0]).get("runtime") or {}).get("command") + + +@pytest.fixture +def initialised(tmp_path: Path) -> Path: + assert _roar(tmp_path, "init", "-n").returncode == 0 + assert _roar(tmp_path, "tracer", "use", "preload").returncode == 0 + return tmp_path + + +def test_a_shebang_script_records_the_script_not_its_interpreter(initialised: Path) -> None: + """The kernel rewrites cmdline to include the shebang interpreter, so /proc + reports ` ./train.sh` for a run of `./train.sh`. + + The interpreter is named directly rather than via `/usr/bin/env` so this + keeps running on macOS, where the system launchers are protected. + """ + script = initialised / "train.sh" + script.write_text(f"#!{sys.executable}\nprint('hi')\n") + script.chmod(0o755) + + run = _roar(initialised, "run", "./train.sh") + + assert run.returncode == 0 + assert _latest_command(initialised) == ["./train.sh"] + + +@_MACOS_PROTECTED_BINARY +def test_a_wrapper_command_is_recorded_as_given(initialised: Path) -> None: + run = _roar(initialised, "run", "env", "-u", "PYTHONPATH", sys.executable, "-c", "pass") + + assert run.returncode == 0 + assert _latest_command(initialised) == [ + "env", + "-u", + "PYTHONPATH", + sys.executable, + "-c", + "pass", + ] + + +@_MACOS_PROTECTED_BINARY +def test_a_short_lived_command_still_records_its_argv(initialised: Path) -> None: + """A process that exits immediately may be a zombie by the time /proc is + read, which is where the recorded argv used to vary with machine load.""" + for _ in range(5): + run = _roar(initialised, "run", "/usr/bin/true") + + assert run.returncode == 0 + assert _latest_command(initialised) == ["/usr/bin/true"] diff --git a/tests/integration/test_reproduce_python_mismatch.py b/tests/integration/test_reproduce_python_mismatch.py new file mode 100644 index 00000000..465d18bb --- /dev/null +++ b/tests/integration/test_reproduce_python_mismatch.py @@ -0,0 +1,78 @@ +"""P0-4 end-to-end: when reproduce cannot provision the recorded Python (no uv) +and the running interpreter differs at major.minor, roar warns loudly, recommends +uv, and asks before continuing — with ``--yes`` (auto_confirm) overriding the +prompt. Declining aborts instead of silently reproducing on the wrong Python. + +These build a REAL venv via ``python -m venv`` (no subprocess mock) so the whole +non-uv path is exercised, including the actual interpreter-version comparison. +""" + +import sys +from unittest.mock import MagicMock + +import pytest + +from roar.execution.reproduction.environment_setup import EnvironmentSetupService + + +def _running_minor() -> str: + return f"{sys.version_info.major}.{sys.version_info.minor}" + + +def _svc(confirm_return: bool | None = None): + presenter = MagicMock() + if confirm_return is not None: + presenter.confirm.return_value = confirm_return + svc = EnvironmentSetupService(presenter=presenter) + svc._use_uv = False # force the `python -m venv` (running-interpreter) path + return svc, presenter + + +def _printed(presenter) -> str: + return " ".join(str(c.args[0]) for c in presenter.print.call_args_list) + + +def test_mismatch_declined_aborts(tmp_path): + """No --yes, user declines the prompt -> RuntimeError, and the warning both + names the mismatch and points at the uv install docs.""" + svc, presenter = _svc(confirm_return=False) + repo = tmp_path / "repo" + repo.mkdir() + with pytest.raises(RuntimeError, match="aborted"): + svc._create_venv(repo, "3.99.0", auto_confirm=False) # 3.99 can't match the runner + out = _printed(presenter) + assert "PYTHON VERSION MISMATCH" in out + assert "docs.astral.sh/uv" in out + presenter.confirm.assert_called_once() + + +def test_mismatch_yes_overrides_prompt_and_builds_venv(tmp_path): + """--yes -> warn loudly but continue without asking; a real venv is built.""" + svc, presenter = _svc() + repo = tmp_path / "repo" + repo.mkdir() + venv = svc._create_venv(repo, "3.99.0", auto_confirm=True) + assert (venv / "pyvenv.cfg").exists() # a genuine venv was created + presenter.confirm.assert_not_called() # --yes means no prompt + assert "PYTHON VERSION MISMATCH" in _printed(presenter) + + +def test_mismatch_confirmed_continues(tmp_path): + """No --yes, user accepts -> continue and build the venv.""" + svc, presenter = _svc(confirm_return=True) + repo = tmp_path / "repo" + repo.mkdir() + venv = svc._create_venv(repo, "3.99.0", auto_confirm=False) + assert (venv / "pyvenv.cfg").exists() + presenter.confirm.assert_called_once() + + +def test_matching_minor_no_prompt_no_warning(tmp_path): + """Recorded minor == running minor -> silent: no warning, no prompt, venv built.""" + svc, presenter = _svc(confirm_return=False) + repo = tmp_path / "repo" + repo.mkdir() + venv = svc._create_venv(repo, f"{_running_minor()}.0", auto_confirm=False) + assert (venv / "pyvenv.cfg").exists() + presenter.confirm.assert_not_called() + assert "MISMATCH" not in _printed(presenter) diff --git a/tests/integrations/glaas/test_client.py b/tests/integrations/glaas/test_client.py index 63d07d2a..a9fd91df 100644 --- a/tests/integrations/glaas/test_client.py +++ b/tests/integrations/glaas/test_client.py @@ -80,6 +80,27 @@ def test_finalize_current_user_private_scope_reports_server_support_gap() -> Non ) +def test_registration_session_label_sync_uses_scoped_route_and_auth() -> None: + client = _optional_auth_client() + client._registration_session_mode = "anonymous_public" + client._registration_session_token = "registration-token" + labels = [{"entity_type": "dag", "session_hash": "a" * 64, "metadata": {}}] + + with patch.object(client, "_request", return_value=({"processed": 1}, None)) as request: + result, error = client.sync_labels_under_registration_session("reg-123", labels) + + assert result == {"processed": 1} + assert error is None + request.assert_called_once_with( + "POST", + "/api/v1/registration-sessions/reg-123/labels/batch", + {"labels": labels}, + auth_header_value="RegistrationSession registration-token", + allow_auth_fallback=False, + ) + assert client._registration_session_token is None + + class TestGlaasClientExceptions: """Test that GlaasClient raises proper exceptions.""" diff --git a/tests/integrations/test_wandb_trackio.py b/tests/integrations/test_wandb_trackio.py index 7d132391..9a6d21b7 100644 --- a/tests/integrations/test_wandb_trackio.py +++ b/tests/integrations/test_wandb_trackio.py @@ -70,6 +70,42 @@ def test_sync_aliases_to_trackio_and_strips_wandb_only_kwargs(): assert "commit" not in calls["log"][1] # wandb-only log kwarg stripped +def test_lerobot_resume_none_is_dropped_before_trackio(): + """trackio.init raises ValueError on ``resume=None``; wandb's default IS None + and lerobot always passes the kwarg (``resume="must" if cfg.resume else None``), + so it dies in init(). The shim must drop a None resume.""" + calls: dict = {} + fake = types.ModuleType("trackio") + fake.init = lambda *a, **k: calls.__setitem__("init", k) or types.SimpleNamespace(summary={}) + fake.log = lambda *a, **k: None + sys.modules["trackio"] = fake + + wandb_trackio.install(environ={"ROAR_WANDB_TO_TRACKIO": "1", "TRACKIO_SPACE_ID": "org/space"}) + import wandb + + wandb.init(project="p", resume=None) # would raise inside trackio without the drop + assert "resume" not in calls["init"] + + +def test_lerobot_log_data_kwarg_is_forwarded_positionally(): + """``wandb.log(data=..., step=...)`` — wandb's first param is named ``data``, + trackio's is ``metrics``, so the keyword form is a TypeError. The shim forwards + it positionally so the metrics actually reach trackio.""" + calls: dict = {} + fake = types.ModuleType("trackio") + fake.init = lambda *a, **k: types.SimpleNamespace(summary={}) + fake.log = lambda *a, **k: calls.__setitem__("log", (a, k)) + sys.modules["trackio"] = fake + + wandb_trackio.install(environ={"ROAR_WANDB_TO_TRACKIO": "1", "TRACKIO_SPACE_ID": "org/space"}) + import wandb + + wandb.log(data={"loss": 0.5}, step=3) + args, kwargs = calls["log"] + assert args and args[0] == {"loss": 0.5} # metrics forwarded positionally + assert "data" not in kwargs # no TypeError-inducing keyword survives + + def test_off_beats_a_configured_space(): fake = types.ModuleType("trackio") fake.init = lambda *a, **k: None @@ -84,3 +120,16 @@ def test_does_not_clobber_existing_wandb(): sys.modules["wandb"] = sentinel wandb_trackio.install(environ={"ROAR_WANDB_TO_TRACKIO": "off"}) assert sys.modules["wandb"] is sentinel + + +def test_noop_wandb_has_a_spec_so_find_spec_does_not_raise(): + """P0-15: a __spec__=None module makes importlib.util.find_spec RAISE, which + crashes `import accelerate` (is_wandb_available) on any credential-free host. + The no-op stub must carry a real spec.""" + import importlib.util + + wandb_trackio.install(environ={"ROAR_WANDB_TO_TRACKIO": "off"}) + stub = sys.modules["wandb"] + assert stub.__spec__ is not None + # This raised ValueError("wandb.__spec__ is None") before the fix. + assert importlib.util.find_spec("wandb") is stub.__spec__ diff --git a/tests/unit/put/test_put_service.py b/tests/unit/put/test_put_service.py index 9aac41c3..48970807 100644 --- a/tests/unit/put/test_put_service.py +++ b/tests/unit/put/test_put_service.py @@ -10,7 +10,7 @@ from roar.application.publish.composite_builder import CompositeArtifactBuilder from roar.application.publish.put_execution import PutService -from roar.application.publish.put_preparation import PreparedPutExecution +from roar.application.publish.put_preparation import DelegatedPutOperation, PreparedPutExecution from roar.application.publish.registration import build_lineage_membership_index_payload from roar.application.publish.results import PutDryRunItem from roar.application.publish.source_resolution import ResolvedSource @@ -83,6 +83,10 @@ def _prepared_put( session_url: str = "https://glaas.ai/dag/session_hash_abc123", destination_type: str = "memory", composite_source_type: str | None = None, + registration_session_id: str | None = None, + registration_session_status: str | None = None, + delegated_put_operation: DelegatedPutOperation | None = None, + lineage: LineageData | None = None, ) -> PreparedPutExecution: resolved: list[ResolvedSource] = [] for source in sources: @@ -120,10 +124,78 @@ def _prepared_put( resolved_sources=resolved, destination_type=destination_type, composite_source_type=composite_source_type, + registration_session_id=registration_session_id, + registration_session_status=registration_session_status, + delegated_put_operation=delegated_put_operation, + lineage=lineage, ) class TestPutService: + def test_put_prepared_uses_the_reserved_lineage_snapshot(self, tmp_path: Path) -> None: + model_file = tmp_path / "model.pt" + model_file.write_bytes(b"model data") + lineage = LineageData( + jobs=[{"job_uid": "reserved-upstream", "command": "python upstream.py"}], + artifacts=[], + artifact_hashes=set(), + ) + collector = MagicMock() + coordinator = _create_mock_coordinator() + service = PutService( + db_context=_create_mock_db(), + backend=MemoryBackend(bucket="test-bucket", prefix="models"), + destination="memory://test-bucket/models", + repo_root=tmp_path, + lineage_collector=collector, + registration_coordinator=coordinator, + ) + + result = service.put_prepared( + prepared=_prepared_put(tmp_path, sources=[model_file], lineage=lineage), + sources=[str(model_file)], + message="publish reserved snapshot", + ) + + assert result.success is True + collector.collect.assert_not_called() + assert coordinator.register_lineage.call_args.kwargs["jobs"] == lineage.jobs + + def test_closed_registration_session_retry_does_not_upload_again(self, tmp_path: Path) -> None: + model_file = tmp_path / "model.pt" + model_file.write_bytes(b"model data") + db = _create_mock_db() + backend = MemoryBackend(bucket="test-bucket", prefix="models") + service = PutService( + db_context=db, + backend=backend, + destination="memory://test-bucket/models", + repo_root=tmp_path, + lineage_collector=MagicMock(), + registration_coordinator=_create_mock_coordinator(), + ) + + with patch.object(service, "_hash_files_batch") as hash_files: + result = service.put_prepared( + prepared=_prepared_put( + tmp_path, + sources=[model_file], + session_hash="authoritative-session-hash", + session_url="https://glaas.example/dag/authoritative-session-hash", + registration_session_id="registration-session-1", + registration_session_status="closed", + ), + sources=[str(model_file)], + message="retry publish model", + ) + + assert result.success is True + assert result.session_hash == "authoritative-session-hash" + assert result.session_url == "https://glaas.example/dag/authoritative-session-hash" + assert result.uploaded_files == [] + hash_files.assert_not_called() + db.jobs.create.assert_not_called() + def test_put_prepared_single_file_creates_job(self, tmp_path: Path) -> None: model_file = tmp_path / "model.pt" model_file.write_bytes(b"model data") @@ -161,6 +233,90 @@ def test_put_prepared_single_file_creates_job(self, tmp_path: Path) -> None: assert call_kwargs["job_type"] == "put" service._db.jobs.add_input.assert_called_once() + def test_active_delegated_retry_reuses_reserved_local_put_job(self, tmp_path: Path) -> None: + model_file = tmp_path / "model.pt" + model_file.write_bytes(b"model data") + db = _create_mock_db() + operation = DelegatedPutOperation( + task_identity="task", + session_id=1, + ordinal=1, + request_fingerprint="fingerprint", + put_job_uid="delegated-put-stable", + ) + db.jobs.get_by_uid.return_value = { + "id": 42, + "job_uid": operation.put_job_uid, + "step_number": 3, + } + coordinator = _create_mock_coordinator() + coordinator.register_lineage_under_registration_session.return_value = ( + BatchRegistrationResult( + session_registered=True, + jobs_created=0, + jobs_failed=0, + artifacts_registered=0, + artifacts_failed=0, + links_created=0, + links_failed=0, + errors=[], + ) + ) + coordinator.job_service.create_job_under_registration_session.return_value = ( + JobRegistrationResult( + success=True, + job_uid="remote-put", + job_id="remote-put", + error=None, + ) + ) + coordinator.job_service.link_job_artifacts_under_registration_session.return_value = ( + JobLinkResult( + success=True, + job_uid="remote-put", + inputs_linked=1, + outputs_linked=0, + error=None, + ) + ) + coordinator.session_service.finalize_registration_session.return_value = MagicMock( + success=True, + session_hash="final-hash", + session_url="https://glaas.example/dag/final-hash", + error=None, + ) + service = PutService( + db_context=db, + backend=MemoryBackend(bucket="test-bucket", prefix="models"), + destination="memory://test-bucket/models", + repo_root=tmp_path, + lineage_collector=MagicMock(), + registration_coordinator=coordinator, + ) + service._lineage_collector.collect.return_value = LineageData( + jobs=[], + artifacts=[], + artifact_hashes=set(), + pipeline={"id": 1}, + ) + + result = service.put_prepared( + prepared=_prepared_put( + tmp_path, + sources=[model_file], + registration_session_id="registration-session", + registration_session_status="active", + delegated_put_operation=operation, + ), + sources=[str(model_file)], + message="retry", + ) + + assert result.success is True + assert result.job_id == 42 + db.jobs.get_by_uid.assert_called_once_with(operation.put_job_uid) + db.jobs.create.assert_not_called() + def test_put_prepared_refreshes_and_syncs_put_job_labels(self, tmp_path: Path) -> None: model_file = tmp_path / "model.pt" model_file.write_bytes(b"model data") diff --git a/tests/unit/test_artifact_registration_phase3_fallback.py b/tests/unit/test_artifact_registration_phase3_fallback.py index a34811b3..09206e1e 100644 --- a/tests/unit/test_artifact_registration_phase3_fallback.py +++ b/tests/unit/test_artifact_registration_phase3_fallback.py @@ -1,6 +1,4 @@ -"""Tests for ArtifactRegistrationService.register_batch_under_registration_session -backwards-compat fallback when talking to a glaas instance that doesn't have the -staged-artifact endpoint (https://github.com/treqs-inc/glaas-api/pull/50).""" +"""Fail-closed tests for scoped artifact staging.""" from unittest.mock import MagicMock @@ -24,10 +22,8 @@ def _artifacts(n=2): ] -def test_404_on_first_batch_silently_skips_phase3(): - """Old glaas without the staged endpoint returns 404; coordinator should - treat it as 'fall back to legacy link-implicit creation' and surface no - error, so `roar register` doesn't fail on every old server.""" +def test_404_on_first_batch_fails_closed(): + """A receiver missing the scoped endpoint is not broker-compatible.""" service, client = _service() client.register_artifacts_batch_under_registration_session.return_value = ( 0, @@ -38,8 +34,8 @@ def test_404_on_first_batch_silently_skips_phase3(): result = service.register_batch_under_registration_session(_artifacts(2), "reg-sess-x") assert result.success_count == 0 - assert result.error_count == 0 - assert result.errors == [] + assert result.error_count == 2 + assert any("404" in error for error in result.errors) def test_non_404_error_still_surfaces(): diff --git a/tests/unit/test_cli_init.py b/tests/unit/test_cli_init.py index a89f2f22..a7e90a7f 100644 --- a/tests/unit/test_cli_init.py +++ b/tests/unit/test_cli_init.py @@ -301,3 +301,75 @@ def test_init_path_uses_target_repo_for_gitignore_updates(tmp_path: Path) -> Non assert caller_gitignore.read_text() == ".roar/\n" assert ".roar/" in target_gitignore.read_text().splitlines() assert (target_repo / ".roar").is_dir() + + +class TestSharedEnvironmentDetection: + """roar sharing the workload's environment means both sets of requirements + must resolve together, and roar's dependencies land in the freeze where + nothing can attribute them (P0-28). The comparison must be against the + interpreter the WORKLOAD would use, not roar's own: under `uv tool` or pipx + roar always sits inside its own prefix, so comparing roar to itself reports + every correctly isolated install as shared -- nagging exactly the users who + took the advice.""" + + def _prefixes(self, monkeypatch, *, roar_prefix, venv=None, conda=None, path_python=None): + from roar.cli.commands import init as init_module + + monkeypatch.setattr(init_module.sys, "prefix", roar_prefix, raising=False) + monkeypatch.delenv("VIRTUAL_ENV", raising=False) + monkeypatch.delenv("CONDA_PREFIX", raising=False) + if venv: + monkeypatch.setenv("VIRTUAL_ENV", venv) + if conda: + monkeypatch.setenv("CONDA_PREFIX", conda) + monkeypatch.setattr(init_module.shutil, "which", lambda _name: path_python, raising=False) + return init_module + + def test_pip_installed_into_the_active_project_venv_is_shared(self, monkeypatch): + init_module = self._prefixes(monkeypatch, roar_prefix="/proj/.venv", venv="/proj/.venv") + assert init_module.roar_shares_this_environment() is True + + def test_a_tool_install_alongside_an_active_project_venv_is_isolated(self, monkeypatch): + """The `uv tool` / pipx layout: roar runs from its own venv.""" + init_module = self._prefixes( + monkeypatch, roar_prefix="/home/u/.local/share/uv/tools/roar-cli", venv="/proj/.venv" + ) + assert init_module.roar_shares_this_environment() is False + + def test_a_tool_install_with_no_venv_active_is_isolated(self, monkeypatch): + """No venv: the workload would run the system python, which is not roar's.""" + init_module = self._prefixes( + monkeypatch, + roar_prefix="/home/u/.local/share/uv/tools/roar-cli", + path_python="/usr/bin/python3", + ) + assert init_module.roar_shares_this_environment() is False + + def test_a_system_install_with_no_venv_is_shared(self, monkeypatch): + init_module = self._prefixes( + monkeypatch, roar_prefix="/usr", path_python="/usr/bin/python3" + ) + assert init_module.roar_shares_this_environment() is True + + def test_a_conda_environment_is_honoured(self, monkeypatch): + init_module = self._prefixes( + monkeypatch, roar_prefix="/opt/conda/envs/proj", conda="/opt/conda/envs/proj" + ) + assert init_module.roar_shares_this_environment() is True + + def test_no_resolvable_interpreter_says_nothing(self, monkeypatch): + init_module = self._prefixes(monkeypatch, roar_prefix="/anything", path_python=None) + assert init_module.roar_shares_this_environment() is False + + def test_detection_never_breaks_init(self, monkeypatch): + """A cosmetic hint must not be able to fail `roar init`.""" + from roar.cli.commands import init as init_module + + def _boom(_name): + raise OSError("PATH exploded") + + monkeypatch.delenv("VIRTUAL_ENV", raising=False) + monkeypatch.delenv("CONDA_PREFIX", raising=False) + monkeypatch.setattr(init_module.shutil, "which", _boom, raising=False) + + assert init_module.roar_shares_this_environment() is False diff --git a/tests/unit/test_coordinator.py b/tests/unit/test_coordinator.py index 790f4027..697bc476 100644 --- a/tests/unit/test_coordinator.py +++ b/tests/unit/test_coordinator.py @@ -398,7 +398,12 @@ def _reg_session_coordinator(batch_counts): def test_full_re_register_short_circuits_to_existing_dag() -> None: coordinator, artifact_service, job_service = _reg_session_coordinator( - {"created": 0, "existing": 0, "already_registered": ["dag-hash-abc"]} + { + "created": 0, + "existing": 0, + "already_registered": ["dag-hash-abc"], + "existing_binding_prepared": True, + } ) result = coordinator.register_lineage_under_registration_session( registration_session_id="reg-1", @@ -407,6 +412,7 @@ def test_full_re_register_short_circuits_to_existing_dag() -> None: artifacts=[{"hashes": [{"algorithm": "blake3", "digest": "in1"}], "size": 1}], ) assert result.already_registered_session_hash == "dag-hash-abc" + assert result.existing_binding_prepared is True assert result.session_registered is True assert result.jobs_failed == 0 # No staging / linking for an already-registered lineage. diff --git a/tests/unit/test_environment_setup.py b/tests/unit/test_environment_setup.py index 63016926..28ca1edb 100644 --- a/tests/unit/test_environment_setup.py +++ b/tests/unit/test_environment_setup.py @@ -697,8 +697,10 @@ def test_prompts_user_for_fallback_when_version_unavailable(self, service, tmp_p "Install available versions instead?", default=True ) - def test_skips_failed_packages_when_user_declines_fallback(self, service, tmp_path): - """When user declines, skip the failed packages with warning.""" + def test_declining_fallback_on_missing_pin_fails(self, service, tmp_path): + """When the user declines the any-version fallback, a recorded pin is + left uninstalled — so the install must FAIL, not silently succeed (P0-1). + Previously this returned True, yielding "Environment ready" + a dead run.""" venv_dir = tmp_path / ".venv" venv_dir.mkdir() repo_dir = tmp_path @@ -716,7 +718,7 @@ def test_skips_failed_packages_when_user_declines_fallback(self, service, tmp_pa auto_confirm=False, ) - assert success is True + assert success is False assert any("exact version not found" in w for w in warnings) def test_identifies_individual_failed_packages(self, service, tmp_path): @@ -806,14 +808,17 @@ def test_recorded_version_none_when_absent(self): pipeline.run_steps = [{"metadata": json.dumps({"packages": {}})}] assert svc._recorded_python_version(pipeline) is None - def test_warn_only_on_minor_mismatch(self): + def test_confirm_only_warns_on_minor_mismatch(self): svc = self._svc() - svc._warn_python_mismatch("3.14.4", "3.14.9") # same minor -> no warn - svc._warn_python_mismatch(None, "3.13.0") # nothing recorded -> no warn + # auto_confirm=True so a mismatch warns-and-continues (no prompt/abort). + svc._confirm_python_mismatch("3.14.4", "3.14.9", auto_confirm=True) # same minor + svc._confirm_python_mismatch(None, "3.13.0", auto_confirm=True) # nothing recorded assert svc._presenter.print.call_count == 0 - svc._warn_python_mismatch("3.14.4", "3.13.14") # minor differs -> warn - msg = svc._presenter.print.call_args[0][0] - assert "Recorded Python was 3.14.4" in msg and "3.13.14" in msg + svc._confirm_python_mismatch("3.14.4", "3.13.14", auto_confirm=True) # minor differs + printed = " ".join(str(c.args[0]) for c in svc._presenter.print.call_args_list) + assert "PYTHON VERSION MISMATCH" in printed + assert "3.14.4" in printed and "3.13.14" in printed + assert "uv" in printed # recommends the deterministic fix def test_uv_venv_pins_recorded_version(self, tmp_path): svc = self._svc() @@ -833,7 +838,7 @@ def fake_run(cmd, **kwargs): args = run.call_args_list[0].args[0] assert args[:2] == ["uv", "venv"] and "--python" in args and "3.14.4" in args # matched the recorded interpreter -> no mismatch warning - assert not any("Recorded Python" in str(c) for c in svc._presenter.print.call_args_list) + assert not any("MISMATCH" in str(c) for c in svc._presenter.print.call_args_list) def test_uv_falls_back_to_minor_then_default_with_warning(self, tmp_path): svc = self._svc() @@ -854,11 +859,13 @@ def fake_run(cmd, **kwargs): return MagicMock(returncode=0) with patch("subprocess.run", side_effect=fake_run): - svc._create_venv(repo_dir, "3.14.4") + # auto_confirm so the resulting mismatch (3.14.4 -> 3.13.14) doesn't prompt. + svc._create_venv(repo_dir, "3.14.4", auto_confirm=True) pythons = [c for c in calls if "--python" in c] assert any("3.14.4" in c for c in pythons) assert any("3.14" in c and "3.14.4" not in c for c in pythons) assert calls[-1] == ["uv", "venv", str(venv_dir)] # bare fallback last - warning = svc._presenter.print.call_args[0][0] - assert "Recorded Python was 3.14.4" in warning and "3.13.14" in warning + printed = " ".join(str(c.args[0]) for c in svc._presenter.print.call_args_list) + assert "PYTHON VERSION MISMATCH" in printed + assert "3.14.4" in printed and "3.13.14" in printed diff --git a/tests/unit/test_pipeline_executor_timeout.py b/tests/unit/test_pipeline_executor_timeout.py new file mode 100644 index 00000000..96dd1b44 --- /dev/null +++ b/tests/unit/test_pipeline_executor_timeout.py @@ -0,0 +1,98 @@ +"""P0-2: per-step timeout is configurable (default none) and, when it fires, +kills the whole process group — not just the shell — so a grandchild workload +(e.g. train.py) can't keep running past the declared failure.""" + +import os +import sys +import time +from unittest.mock import MagicMock + +from roar.execution.reproduction.pipeline_executor import PipelineExecutor + + +def _executor(step_timeout=None): + ex = PipelineExecutor(roar_executable="/bin/true", step_timeout=step_timeout) + ex._print = lambda *_: None + return ex + + +def _drive(ex, wrapped_command, environment): + """Run one step, forcing the wrapped command and a clean env.""" + ex._wrap_with_roar = lambda *a, **k: wrapped_command + ex._prepare_environment = lambda *a, **k: dict(os.environ) + step = {"command": "x", "metadata": {}} + return ex._run_step(step, environment, is_build=False) + + +def test_default_step_timeout_is_none(): + assert PipelineExecutor()._step_timeout is None + + +def test_quick_command_succeeds_with_no_timeout(tmp_path): + ex = _executor(step_timeout=None) + env = MagicMock(repo_dir=tmp_path) + assert _drive(ex, f'{sys.executable} -c "pass"', env) is True + + +def test_failing_command_returns_false(tmp_path): + ex = _executor(step_timeout=None) + env = MagicMock(repo_dir=tmp_path) + assert _drive(ex, f'{sys.executable} -c "raise SystemExit(3)"', env) is False + + +def test_timeout_kills_the_whole_process_group(tmp_path): + """A shell child that spawns a long-lived grandchild must be fully reaped on + timeout. Before the fix (shell=True + subprocess.run timeout), only the shell + died and the grandchild ran on — a false failure plus a silent GPU-cost leak. + + We prove the kill by *liveness*, not by waiting out a sleep: the grandchild + bumps a counter file every 50ms. Once the step times out and the group is + SIGKILLed, the counter must stop advancing. (A plain ``os.kill(pid, 0)`` + check is unreliable here — a killed-but-unreaped grandchild is a zombie, for + which ``os.kill`` still reports "alive".) No long fixed sleep, so this stays + ~1.5s on the slow macOS lane.""" + heartbeat = tmp_path / "grandchild.heartbeat" + # Real files, not `python -c` payloads — the -c escaping for a multi-line + # loop is a trap (a single broken literal makes the child a no-op and the + # test silently inconclusive). + grand = tmp_path / "grand.py" + grand.write_text( + "import time\n" + "i = 0\n" + "while True:\n" + f" open({str(heartbeat)!r}, 'w').write(str(i))\n" + " i += 1\n" + " time.sleep(0.05)\n" + ) + child = tmp_path / "child.py" + child.write_text( + "import subprocess, sys, time\n" + f"subprocess.Popen([sys.executable, {str(grand)!r}])\n" + "time.sleep(30)\n" + ) + + ex = _executor(step_timeout=1) + env = MagicMock(repo_dir=tmp_path) + + start = time.monotonic() + result = _drive(ex, f"{sys.executable} {child}", env) + elapsed = time.monotonic() - start + + assert result is False + assert elapsed < 8, f"timeout should fire ~1s, took {elapsed:.1f}s" + + # The grandchild starts heartbeating well within the 1s timeout. + deadline = start + 3 + while not heartbeat.exists() and time.monotonic() < deadline: + time.sleep(0.05) + assert heartbeat.exists(), "grandchild never started — test inconclusive" + + # If the whole group was killed the counter is frozen; a survivor keeps + # advancing it. Two reads 0.5s apart (>> the 50ms heartbeat) settle it. + before = heartbeat.read_text() + time.sleep(0.5) + after = heartbeat.read_text() + assert before == after, ( + f"grandchild kept running after the timeout ({before!r} -> {after!r}) " + "— process group not killed" + ) diff --git a/tests/unit/test_publish_auth_context.py b/tests/unit/test_publish_auth_context.py index a6c1d165..5e80fa4b 100644 --- a/tests/unit/test_publish_auth_context.py +++ b/tests/unit/test_publish_auth_context.py @@ -96,6 +96,35 @@ def test_private_publish_without_binding_uses_current_user_scope_request( } +def test_delegated_publish_ignores_ambient_auth_and_resolves_private_context( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("ROAR_DELEGATED_AUTH", "1") + monkeypatch.setenv("ROAR_DELEGATED_USER_SUB", "sub-123") + monkeypatch.setenv("ROAR_DELEGATED_DB_USER_ID", "user-123") + monkeypatch.setenv("ROAR_DELEGATED_CREATOR_IDENTITY", "treqs:user:sub-123") + monkeypatch.setenv("ROAR_DELEGATED_OWNER_ID", "org-123") + monkeypatch.setenv("ROAR_DELEGATED_OWNER_TYPE", "organization") + monkeypatch.setenv("ROAR_DELEGATED_PROJECT_ID", "project-456") + monkeypatch.setenv("ROAR_DELEGATED_VISIBILITY", "private") + + with patch("roar.publish_auth.load_auth_state", side_effect=AssertionError("must not load")): + context = load_publish_auth_context( + start_dir=tmp_path, + allow_public_without_binding=False, + ) + + assert context.access_token is None + assert context.delegated_auth_available + assert context.creator_identity == "treqs:user:sub-123" + assert context.scope_request == { + "owner_id": "org-123", + "owner_type": "organization", + "project_id": "project-456", + "visibility": "private", + } + + def test_public_scope_uses_current_user_public_scope_request(tmp_path: Path) -> None: config_dir = tmp_path / ".roar" config_dir.mkdir(parents=True) diff --git a/tests/unit/test_publish_intent.py b/tests/unit/test_publish_intent.py index a247a9ef..79ee7f4f 100644 --- a/tests/unit/test_publish_intent.py +++ b/tests/unit/test_publish_intent.py @@ -51,6 +51,32 @@ def test_unset_logged_in_defaults_private(): assert not out.defaulted_anonymous +def test_delegated_task_defaults_private_without_auth_file(monkeypatch): + monkeypatch.setenv("ROAR_DELEGATED_AUTH", "1") + monkeypatch.setenv("ROAR_DELEGATED_VISIBILITY", "private") + with ( + patch("roar.scope_config.load_repo_scope", return_value=None), + patch("roar.auth_store.load_auth_state", return_value=None), + patch("roar.integrations.config.config_get", return_value=False), + ): + out = resolve_publish_intent(None, False) + + assert not out.public and not out.anonymous + + +def test_delegated_task_uses_frozen_visibility_over_repo_and_flags(monkeypatch): + monkeypatch.setenv("ROAR_DELEGATED_AUTH", "1") + monkeypatch.setenv("ROAR_DELEGATED_VISIBILITY", "public") + with patch( + "roar.scope_config.load_repo_scope", + return_value=SimpleNamespace(mode="anonymous", visibility=None), + ) as load_repo_scope: + out = resolve_publish_intent(public=False, anonymous=True) + + assert out.public and not out.anonymous + load_repo_scope.assert_not_called() + + def test_unset_not_logged_in_defaults_anonymous_with_flag(): out = _resolve(scope=None, logged_in=False) assert out.public and out.anonymous diff --git a/tests/unit/test_register_cli.py b/tests/unit/test_register_cli.py index 69f256b7..9af94ba7 100644 --- a/tests/unit/test_register_cli.py +++ b/tests/unit/test_register_cli.py @@ -363,9 +363,9 @@ def test_register_cli_renders_warnings_above_summary(tmp_path: Path) -> None: def test_register_cli_no_target_defaults_to_active_session(tmp_path: Path) -> None: """`roar register` with no target registers the whole active session. - It resolves the active session's canonical hash and passes it as the target - so the session_hash collection path runs (the full DAG, incl. downstream - steps), not an artifact's upstream-only ancestry. + It resolves the active session's canonical hash only for confirmation, then + preserves target=None so the application selects the active session after + publish bootstrap (the full DAG, including downstream steps). """ runner = CliRunner() session_hash = "c" * 64 @@ -381,7 +381,7 @@ def test_register_cli_no_target_defaults_to_active_session(tmp_path: Path) -> No assert result.exit_code == 0, result.output request = mock_register.call_args.args[0] - assert request.target == session_hash + assert request.target is None def test_register_cli_no_target_without_active_session_errors(tmp_path: Path) -> None: @@ -482,7 +482,7 @@ def test_register_cli_accepts_defaulted_active_session_publish_prompt(tmp_path: assert result.exit_code == 0, result.output request = mock_register.call_args.args[0] - assert request.target == session_hash + assert request.target is None def test_register_cli_defaulted_active_session_prompt_has_no_in_flight_warning_by_default( @@ -562,7 +562,7 @@ def test_register_cli_yes_skips_defaulted_active_session_prompt(tmp_path: Path) assert result.exit_code == 0, result.output assert "Publish the whole active session?" not in result.output request = mock_register.call_args.args[0] - assert request.target == session_hash + assert request.target is None def test_register_cli_dry_run_skips_defaulted_active_session_prompt(tmp_path: Path) -> None: @@ -583,7 +583,7 @@ def test_register_cli_dry_run_skips_defaulted_active_session_prompt(tmp_path: Pa assert result.exit_code == 0, result.output assert "Publish the whole active session?" not in result.output request = mock_register.call_args.args[0] - assert request.target == session_hash + assert request.target is None assert request.dry_run is True diff --git a/tests/unit/test_register_secrets.py b/tests/unit/test_register_secrets.py index 68b510dc..49c56449 100644 --- a/tests/unit/test_register_secrets.py +++ b/tests/unit/test_register_secrets.py @@ -142,3 +142,26 @@ def test_filter_git_context_secrets_without_filter_returns_context_unchanged() - assert filtered is context assert detections == [] + + +def test_hf_token_env_vars_are_redacted_by_default() -> None: + """P0-16: HF_TOKEN / HUGGING_FACE_HUB_TOKEN must be in the built-in env-var + redaction defaults so a live token doesn't reach a published DAG.""" + from roar.filters.omit import OmitFilter + from roar.integrations.config.raw import _DEFAULT_REGISTRATION_OMIT + + names = OmitFilter(_DEFAULT_REGISTRATION_OMIT).env_var_names + assert "HF_TOKEN" in names + assert "HUGGING_FACE_HUB_TOKEN" in names + + +def test_hf_token_value_regex_catches_length_variants() -> None: + """P0-16: the always-on HF value pattern is hf_[A-Za-z0-9]{20,}, not exact-34, + so a token that isn't 34 chars is still caught (defense-in-depth).""" + from roar.filters.omit import OmitFilter + + f = OmitFilter({}) # BUILTIN_PATTERNS apply regardless of config + ids_30 = {m.pattern_id for m in f.detect_secrets("token=hf_" + "A" * 30)} + ids_34 = {m.pattern_id for m in f.detect_secrets("token=hf_" + "A" * 34)} + assert "huggingface_token" in ids_30 # 30 chars: the old {34} regex MISSED this + assert "huggingface_token" in ids_34 diff --git a/tests/unit/test_register_service.py b/tests/unit/test_register_service.py index 0d5e9880..828b352a 100644 --- a/tests/unit/test_register_service.py +++ b/tests/unit/test_register_service.py @@ -1,5 +1,6 @@ """Focused unit tests for RegisterService registration mechanics.""" +import json from pathlib import Path from unittest.mock import MagicMock, patch @@ -91,6 +92,57 @@ def test_order_jobs_for_registration_puts_parent_before_child(self) -> None: assert [job["job_uid"] for job in ordered] == ["parent-uid", "child-uid"] + def test_closed_delegated_publication_reuses_persisted_remote_job_uids( + self, tmp_path: Path + ) -> None: + remote_uid = "remote-job-authoritative" + prepared = PreparedRegisterExecution( + git_context=_git_context(tmp_path), + session_id=1, + session_hash="f" * 64, + session_url="https://glaas.example/dag/existing", + git_tag_name=None, + git_tag_repo_root=None, + registration_session_id="rs-closed", + registration_session_status="closed", + ) + with ( + patch("roar.application.publish.register_execution.config_get", return_value=False), + patch( + "roar.application.publish.register_execution.create_database_context" + ) as mock_ctx, + patch( + "roar.application.publish.register_execution.sync_publish_labels", + return_value=1, + ) as sync_labels, + ): + db_ctx = MagicMock() + db_ctx.__enter__ = MagicMock(return_value=db_ctx) + db_ctx.__exit__ = MagicMock(return_value=None) + db_ctx.sessions.get.return_value = { + "metadata": json.dumps( + {"roar": {"remote_publication": {"glaas": {"jobs": {"job-local": remote_uid}}}}} + ) + } + mock_ctx.return_value = db_ctx + + result = self.service.register_prepared_lineage( + lineage=_lineage_data(jobs=[{"id": 1, "job_uid": "job-local"}]), + roar_dir=tmp_path / ".roar", + artifact_hash="", + dry_run=False, + as_blake3=False, + skip_confirmation=True, + confirm_callback=None, + prepared=prepared, + ) + + assert result.success is True + assert result.labels_synced == 1 + synced_jobs = sync_labels.call_args.kwargs["jobs"] + assert synced_jobs[0]["job_uid"] == "job-local" + assert synced_jobs[0]["remote_job_uid"] == remote_uid + def test_normalize_jobs_for_registration_filters_known_ray_noise_jobs(self) -> None: submit_job = { "id": 1, @@ -463,3 +515,73 @@ def test_register_prepared_lineage_sends_redacted_git_context(self, tmp_path: Pa == "https://user:[REDACTED]@github.com/org/repo.git" ) assert "supersecrettoken123" not in str(finalize_call) + + def test_existing_delegated_binding_is_finalized_before_scoped_label_sync( + self, tmp_path: Path + ) -> None: + from roar.core.interfaces.registration import SessionRegistrationResult + + existing_hash = "e" * 64 + mock_coordinator = MagicMock() + mock_coordinator.register_lineage_under_registration_session.return_value = ( + BatchRegistrationResult( + session_registered=True, + jobs_created=0, + jobs_failed=0, + artifacts_registered=0, + artifacts_failed=0, + links_created=0, + links_failed=0, + errors=[], + already_registered_session_hash=existing_hash, + existing_binding_prepared=True, + ) + ) + mock_coordinator.session_service.finalize_registration_session.return_value = ( + SessionRegistrationResult( + success=True, + session_hash=existing_hash, + session_url=f"https://glaas.example/dag/{existing_hash}", + ) + ) + service = RegisterService(glaas_client=MagicMock(), coordinator=mock_coordinator) + prepared = PreparedRegisterExecution( + git_context=GitContext(repo=None, commit=None, branch=None), + session_id=None, + session_hash="local-hash", + session_url=None, + git_tag_name=None, + git_tag_repo_root=None, + registration_session_id="rs-existing", + ) + + with patch("roar.application.publish.register_execution.config_get", return_value=False): + result = service.register_prepared_lineage( + lineage=_lineage_data( + jobs=[ + { + "id": 1, + "job_uid": "job-existing", + "step_number": 1, + "timestamp": 10.0, + "command": "python train.py", + } + ], + artifacts=[], + artifact_hashes=set(), + ), + roar_dir=tmp_path / ".roar", + artifact_hash=None, + dry_run=False, + as_blake3=False, + skip_confirmation=True, + confirm_callback=None, + prepared=prepared, + ) + + assert result.success is True + assert result.session_hash == existing_hash + mock_coordinator.session_service.finalize_registration_session.assert_called_once_with( + registration_session_id="rs-existing", + git_context=prepared.git_context, + ) diff --git a/tests/unit/test_reproduce_pin_failure.py b/tests/unit/test_reproduce_pin_failure.py new file mode 100644 index 00000000..3e12cfd3 --- /dev/null +++ b/tests/unit/test_reproduce_pin_failure.py @@ -0,0 +1,124 @@ +"""P0-1: reproduce must FAIL (not report success) when a recorded pip pin can't +install, and must offer a debuggable export of the pins.""" + +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +from roar.execution.reproduction.installers import PythonPackageInstaller + + +def _installer() -> PythonPackageInstaller: + return PythonPackageInstaller(use_uv=False, print_fn=lambda *_: None) + + +def _pip(rc: int, stderr: str = "") -> SimpleNamespace: + return SimpleNamespace(returncode=rc, stderr=stderr, stdout="") + + +def _fake_run_pip(*, fail_exact: bool = True, fail_fallback: bool = False, all_ok: bool = False): + def run_pip(venv_dir, repo_dir, args, show_output=False): + if all_ok: + return _pip(0) + # Exact-pin installs carry "==" and the per-package probe carries "--dry-run"; + # the recovery install of an *unversioned* name is the fallback. + if any("==" in a for a in args) or "--dry-run" in args: + return _pip(1 if fail_exact else 0, "No matching distribution") + return _pip(1 if fail_fallback else 0, "No matching distribution") + + return run_pip + + +def test_returns_false_when_pin_unresolvable_and_no_any_version(monkeypatch): + inst = _installer() + monkeypatch.setattr(inst, "_run_pip", _fake_run_pip(fail_exact=True)) + ok, _warnings = inst.install_packages( + Path("/venv"), + ["yanked-pkg==9.9.9"], + Path("/repo"), + auto_confirm=True, + allow_any_version=False, + ) + assert ok is False # was True before the fix -> "Environment ready" + dead run + + +def test_returns_false_when_any_version_fallback_also_fails(monkeypatch): + inst = _installer() + monkeypatch.setattr(inst, "_run_pip", _fake_run_pip(fail_exact=True, fail_fallback=True)) + ok, _warnings = inst.install_packages( + Path("/venv"), + ["private-pkg==1.0"], + Path("/repo"), + auto_confirm=True, + allow_any_version=True, + ) + assert ok is False + + +def test_returns_false_when_individually_resolvable_pins_conflict_in_combination(monkeypatch): + """The recovery path: the batch install fails, every pin passes its per-package + --dry-run (resolvable ALONE), but the combined re-install conflicts and pip + leaves an incomplete venv. Before the guard, that combined install's return + code was discarded -> "Environment ready" over a venv missing most packages + (MiniMind-O: 35 of 105). Must now fail honestly.""" + inst = _installer() + + def run_pip(venv_dir, repo_dir, args, show_output=False): + if "--dry-run" in args: + return _pip(0) # each pin resolves on its own + if any("==" in a for a in args): + return _pip(1, "ResolutionImpossible") # ...but not together + return _pip(0) + + monkeypatch.setattr(inst, "_run_pip", run_pip) + ok, _warnings = inst.install_packages( + Path("/venv"), + ["torch==2.7.0", "numpy==2.0.0"], + Path("/repo"), + auto_confirm=True, + allow_any_version=False, + ) + assert ok is False + + +def test_returns_true_when_all_pins_install(monkeypatch): + inst = _installer() + monkeypatch.setattr(inst, "_run_pip", _fake_run_pip(all_ok=True)) + ok, warnings = inst.install_packages( + Path("/venv"), ["numpy==2.0.0"], Path("/repo"), auto_confirm=True + ) + assert ok is True + assert warnings == [] + + +def test_any_version_recovery_returns_true_with_warning(monkeypatch): + inst = _installer() + monkeypatch.setattr(inst, "_run_pip", _fake_run_pip(fail_exact=True, fail_fallback=False)) + ok, warnings = inst.install_packages( + Path("/venv"), + ["driftable-pkg==1.0"], + Path("/repo"), + auto_confirm=True, + allow_any_version=True, # the bypass: install an available version + ) + assert ok is True + assert any("driftable-pkg" in w for w in warnings) + + +def test_export_requirements_writes_recorded_pins(tmp_path): + from roar.application.reproduce.service import _export_pip_requirements + + pipeline = SimpleNamespace( + build_steps=[], + run_steps=[{"metadata": {"packages": {"pip": {"numpy": "2.0.0", "torch": "2.7.0"}}}}], + artifact_hash="abc123def456", + session_hash=None, + ) + out = MagicMock() + dest = tmp_path / "req.txt" + _export_pip_requirements(pipeline, str(dest), out) + + text = dest.read_text() + assert "numpy==2.0.0" in text + assert "torch==2.7.0" in text + assert text.lstrip().startswith("#") # has the debug header diff --git a/tests/unit/test_runtime_dependencies.py b/tests/unit/test_runtime_dependencies.py new file mode 100644 index 00000000..e29149ec --- /dev/null +++ b/tests/unit/test_runtime_dependencies.py @@ -0,0 +1,19 @@ +"""Guard roar's declared runtime dependencies for shipped code paths.""" + +from __future__ import annotations + +from importlib import metadata + + +def test_huggingface_hub_is_a_runtime_dependency(): + """P0-19: `roar put hf://` imports huggingface_hub, so it must be a RUNTIME + dependency, not a dev-only extra. When it lived under the `dev` extra, + `uv tool install ` (the mandated install path) omitted it and + `roar put hf://` died with ModuleNotFoundError *after* every step succeeded. + """ + reqs = metadata.requires("roar-cli") or [] + hf = [r for r in reqs if r.lower().replace("_", "-").startswith("huggingface-hub")] + assert hf, f"huggingface_hub missing from roar-cli requirements: {reqs}" + # It must not be gated behind an extra (e.g. `; extra == "dev"`). + behind_extra = [r for r in hf if "extra ==" in r or "extra==" in r] + assert not behind_extra, f"huggingface_hub is still behind an extra: {behind_extra}" diff --git a/tests/unit/test_schema_parent_job_uid.py b/tests/unit/test_schema_parent_job_uid.py index 8ac03c7e..4e63592c 100644 --- a/tests/unit/test_schema_parent_job_uid.py +++ b/tests/unit/test_schema_parent_job_uid.py @@ -103,6 +103,39 @@ def test_insert_job_with_null_parent_job_uid_succeeds() -> None: assert row["parent_job_uid"] is None +def test_run_migrations_adds_stable_put_job_uid_to_v4_reservations() -> None: + conn = _create_legacy_db() + conn.executescript( + """ + CREATE TABLE sessions (id INTEGER PRIMARY KEY); + INSERT INTO sessions (id) VALUES (7); + CREATE TABLE delegated_put_operations ( + task_identity TEXT NOT NULL, + session_id INTEGER NOT NULL, + ordinal INTEGER NOT NULL, + request_fingerprint TEXT NOT NULL, + status TEXT NOT NULL, + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + completed_at REAL, + PRIMARY KEY (task_identity, session_id) + ); + INSERT INTO delegated_put_operations ( + task_identity, session_id, ordinal, request_fingerprint, + status, created_at, updated_at + ) VALUES ('abcdef0123456789', 7, 2, 'fingerprint', 'pending', 1, 1); + PRAGMA user_version = 4; + """ + ) + + run_migrations(conn) + + row = conn.execute("SELECT put_job_uid FROM delegated_put_operations").fetchone() + assert row is not None + assert row["put_job_uid"] == "delegated-put-abcdef0123456789-7-2" + assert conn.execute("PRAGMA user_version").fetchone()[0] == 5 + + def test_create_database_context_migrates_parent_job_uid_for_legacy_db(tmp_path: Path) -> None: roar_dir = tmp_path / ".roar" roar_dir.mkdir() diff --git a/tests/unit/test_tracer_data_loader.py b/tests/unit/test_tracer_data_loader.py index 8f4e9945..ff37d3b6 100644 --- a/tests/unit/test_tracer_data_loader.py +++ b/tests/unit/test_tracer_data_loader.py @@ -154,6 +154,21 @@ def test_preserves_thread_aware_file_contract_fields(self, tmp_path: Path) -> No class TestLoadPythonData: + def test_missing_log_is_distinct_from_complete_empty_capture(self, tmp_path: Path) -> None: + missing = DataLoaderService().load_python_data(None) + assert missing.capture_status == "missing" + + log_path = tmp_path / "inject-log.json" + _write_json(log_path, {}) + complete = DataLoaderService().load_python_data(str(log_path)) + assert complete.capture_status == "complete" + + def test_invalid_log_is_reported(self, tmp_path: Path) -> None: + log_path = tmp_path / "inject-log.json" + log_path.write_text("{not-json", encoding="utf-8") + + assert DataLoaderService().load_python_data(str(log_path)).capture_status == "invalid" + def test_python_identity_keys_flow_through(self, tmp_path: Path) -> None: """python_version / python_implementation make it from JSON into the model.""" log_path = tmp_path / "inject-log.json" @@ -211,7 +226,10 @@ def test_writer_reader_roundtrip_carries_python_identity(self, tmp_path: Path) - writer drops the key or the reader doesn't extract it, the loaded model's python_version is empty. """ - from roar.execution.runtime.inject.tracker import RuntimeInjectionTracker + from roar.execution.runtime.inject.tracker import ( + RuntimeInjectionTracker, + merge_inject_logs, + ) log_path = tmp_path / "inject-log.json" @@ -226,6 +244,7 @@ def handle_import(self, module_name, module) -> None: inject_dir=str(tmp_path / "inject"), ) tracker.write_log() + merge_inject_logs(str(log_path)) # write_log writes a per-PID shard; merge -> canonical data = DataLoaderService().load_python_data(str(log_path)) diff --git a/uv.lock b/uv.lock index 5597e453..c7b61c3e 100644 --- a/uv.lock +++ b/uv.lock @@ -15,6 +15,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] +[[package]] +name = "anyio" +version = "4.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, +] + [[package]] name = "blake3" version = "1.0.8" @@ -320,14 +334,14 @@ wheels = [ [[package]] name = "click" -version = "8.3.1" +version = "8.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] [[package]] @@ -550,6 +564,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, ] +[[package]] +name = "filelock" +version = "3.32.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/64/a02e6765de08964ed371eca577870593245afc9dfac16d037de7c10d18e6/filelock-3.32.3.tar.gz", hash = "sha256:0ffa185a3540854c95caa7fa76b76cb219d907415e2c5dc9af25fd970563487f", size = 218135, upload-time = "2026-08-13T16:00:05.577Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/8e/50f46a9c0ce8d2861a394c1347caae037ea0431d2f67d7feb151cbc4649a/filelock-3.32.3-py3-none-any.whl", hash = "sha256:7f0ca4bcc0e181c60dbbd8aa9ab5b120ebb99e4e064e83636340056f833a1f09", size = 98901, upload-time = "2026-08-13T16:00:03.974Z" }, +] + +[[package]] +name = "fsspec" +version = "2026.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, +] + [[package]] name = "google-api-core" version = "2.29.0" @@ -718,6 +750,87 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4f/dc/041be1dff9f23dac5f48a43323cd0789cb798342011c19a248d9c9335536/greenlet-3.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c10513330af5b8ae16f023e8ddbfb486ab355d04467c4679c5cfe4659975dd9", size = 1676034, upload-time = "2025-12-04T14:27:33.531Z" }, ] +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "hf-xet" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/ab/522a2ab67f27971a9d48ca666d4fca85ef7d5282d142e31fd087e27b1bbe/hf_xet-1.6.0.tar.gz", hash = "sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef", size = 920527, upload-time = "2026-08-03T22:33:13.243Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/62/3c062f593bd92ef4e77a0ef39541e3d82a0a1d3947c8a777a02a13a27828/hf_xet-1.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d", size = 4074584, upload-time = "2026-08-03T22:32:47.364Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1e/c0ad437dd267a8e435bef594acf781bbc3874ff0b6435b4962d03ecf7cc4/hf_xet-1.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675", size = 3867381, upload-time = "2026-08-03T22:32:49.049Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ee/7c0d7b6ab336167531b1c30af2af003f054af4c749becbd7209ae33a77c3/hf_xet-1.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b", size = 4453982, upload-time = "2026-08-03T22:32:50.568Z" }, + { url = "https://files.pythonhosted.org/packages/63/06/ad8eab1c9525246650cbaa821caa3cdbaca734ab1a5b8c91bea09cbd8d69/hf_xet-1.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522", size = 4249445, upload-time = "2026-08-03T22:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/d8/26/1eee8aedb0dafc1ab9717dc9ac602cde33361b232dc06803f1f6ed18b58c/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e", size = 4451099, upload-time = "2026-08-03T22:32:54.114Z" }, + { url = "https://files.pythonhosted.org/packages/67/57/0b88af1f194ab6c9c650547d9cc06bfeaab836ae4dcdb331676bfb8be95a/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9", size = 4664712, upload-time = "2026-08-03T22:32:55.547Z" }, + { url = "https://files.pythonhosted.org/packages/53/a0/26b717a9d1840e8abf48dcec64b5ed8fbe472671d38ad28d30e147132b33/hf_xet-1.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338", size = 4025906, upload-time = "2026-08-03T22:32:57.391Z" }, + { url = "https://files.pythonhosted.org/packages/49/f6/4a9966633c6fef83af997e2cff68ec1963676d412bdfd096df2a93b8e185/hf_xet-1.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765", size = 3849221, upload-time = "2026-08-03T22:32:59.123Z" }, + { url = "https://files.pythonhosted.org/packages/a2/50/7afa2c9c787405864fc47a0d1bbc02c62e9101947ed43c1f43899fc7d91d/hf_xet-1.6.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d", size = 4071729, upload-time = "2026-08-03T22:33:00.721Z" }, + { url = "https://files.pythonhosted.org/packages/4b/69/55b8dcf636142ae660fec1869fcac14c4da2e8412e14d6eee1523be77e9f/hf_xet-1.6.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a", size = 3876287, upload-time = "2026-08-03T22:33:02.251Z" }, + { url = "https://files.pythonhosted.org/packages/67/4e/a28359bf1c1ecf11eba22123168c138698f7cb576ac678f5a2e16cd5da08/hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f", size = 4464663, upload-time = "2026-08-03T22:33:03.802Z" }, + { url = "https://files.pythonhosted.org/packages/9a/69/1f0cbc2fb22ae6082d094f743d1b8945a3f36f6089cb95f42b7ee348cda7/hf_xet-1.6.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7", size = 4262538, upload-time = "2026-08-03T22:33:05.287Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3a/4f4f2301ade26e404462d3336fa11f7958d914cabbabdd6e03c3c5d5658c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb", size = 4460520, upload-time = "2026-08-03T22:33:06.81Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/311725e2a905534dfee2dcb5b08414f249147f1f12252bfc2bd24caa075c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c", size = 4675937, upload-time = "2026-08-03T22:33:08.616Z" }, + { url = "https://files.pythonhosted.org/packages/98/b7/8c59a66d15205024662f1d66968136f13893f96df1ddc5087e2e281fc95f/hf_xet-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b", size = 4033128, upload-time = "2026-08-03T22:33:10.171Z" }, + { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "huggingface-hub" +version = "1.28.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/ae/222a91937ebee7f62c0ca8f5ee0afd97577caf24c0abb927d1f5c7e9f6d2/huggingface_hub-1.28.0.tar.gz", hash = "sha256:46a2e950c09234de54093d587d1675382f0d08dbd600d9fb599b5932f5b2c6cb", size = 959609, upload-time = "2026-08-18T12:27:15.101Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/0e/eafef18f1a75e125e68395db21131db0cf868a128ecd2fce69b4df6c584b/huggingface_hub-1.28.0-py3-none-any.whl", hash = "sha256:58a8bacb03072edfc38067065e9dc24bbb34805410fcd36a1632de0b329660bb", size = 793202, upload-time = "2026-08-18T12:27:12.719Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -1429,7 +1542,7 @@ wheels = [ [[package]] name = "roar-cli" -version = "0.3.7" +version = "0.4.4" source = { editable = "." } dependencies = [ { name = "blake3" }, @@ -1450,6 +1563,7 @@ dependencies = [ dev = [ { name = "boto3" }, { name = "google-cloud-storage" }, + { name = "huggingface-hub" }, { name = "mypy" }, { name = "pytest" }, { name = "pytest-cov" }, @@ -1467,6 +1581,7 @@ requires-dist = [ { name = "cryptography", specifier = ">=42.0.0" }, { name = "dependency-injector", specifier = ">=4.40.0" }, { name = "google-cloud-storage", marker = "extra == 'dev'", specifier = ">=2.10.0" }, + { name = "huggingface-hub", marker = "extra == 'dev'", specifier = ">=0.20.0" }, { name = "msgpack", specifier = ">=1.0.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.13.0" }, { name = "pydantic", specifier = ">=2.0.0" }, @@ -1659,6 +1774,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/b8/0135fadc89e73be292b473cb820b4f5a08197779206b33191e801feeae40/tomli-2.3.0-py3-none-any.whl", hash = "sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b", size = 14408, upload-time = "2025-10-08T22:01:46.04Z" }, ] +[[package]] +name = "tqdm" +version = "4.70.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/3b/6c24bec5be5e743ffd99576daa5cc077722fc7d5bbc00bd133fa0c698dc6/tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220", size = 795438, upload-time = "2026-07-27T11:33:15.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953", size = 80184, upload-time = "2026-07-27T11:33:13.167Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0"