From 38e7b849bc5e27d43f70c0e73ff9056a560c2f8a Mon Sep 17 00:00:00 2001 From: Merengues0x2A Date: Mon, 13 Jul 2026 15:46:18 +0800 Subject: [PATCH 1/4] feat: add durable background experiment runs --- CONTEXT.md | 35 ++ antelab/runs/__init__.py | 27 + antelab/runs/artifacts.py | 272 ++++++++++ antelab/runs/errors.py | 25 + antelab/runs/manager.py | 52 ++ antelab/runs/model.py | 162 ++++++ antelab/runs/ownership.py | 14 + antelab/runs/recovery.py | 38 ++ antelab/runs/repository.py | 464 ++++++++++++++++++ antelab/runs/sqlite_repository.py | 356 ++++++++++++++ antelab/runs/worker.py | 240 +++++++++ docs/adr/0001-background-experiment-runs.md | 11 + ...background-experiment-runs-architecture.md | 317 ++++++++++++ ...antelab-background-experiment-runs.goal.md | 89 ++++ .../background-runs-precommit-closure.goal.md | 35 ++ .../background-runs-readiness-repair.goal.md | 17 + .../round-01.md | 30 ++ .../round-02.md | 29 ++ .../round-03.md | 30 ++ .../round-04.md | 76 +++ ...kground-runs-precommit-closure-round-01.md | 18 + ...kground-runs-precommit-closure-round-02.md | 21 + ...ckground-runs-readiness-repair-round-01.md | 18 + ...ckground-runs-readiness-repair-round-02.md | 14 + ...ckground-runs-readiness-repair-round-03.md | 20 + scripts/validate_background_runs_readiness.py | 136 +++++ ...ntelab-background-experiment-runs.STATE.md | 42 ++ ...background-runs-precommit-closure.STATE.md | 24 + .../background-runs-readiness-repair.STATE.md | 30 ++ tests/manifests/background-run-scenarios.txt | 23 + .../background-runs-round-one.sha256 | 34 ++ tests/runs/conftest.py | 28 ++ tests/runs/test_adapter_contracts.py | 175 +++++++ tests/runs/test_durable_adapters.py | 212 ++++++++ tests/runs/test_experiment_runs.py | 54 ++ tests/runs/test_inprocess_engine.py | 89 ++++ tests/runs/test_run_repository.py | 178 +++++++ tests/runs/test_run_worker.py | 268 ++++++++++ tests/test_background_runs_verifier.py | 86 ++++ 39 files changed, 3789 insertions(+) create mode 100644 CONTEXT.md create mode 100644 antelab/runs/__init__.py create mode 100644 antelab/runs/artifacts.py create mode 100644 antelab/runs/errors.py create mode 100644 antelab/runs/manager.py create mode 100644 antelab/runs/model.py create mode 100644 antelab/runs/ownership.py create mode 100644 antelab/runs/recovery.py create mode 100644 antelab/runs/repository.py create mode 100644 antelab/runs/sqlite_repository.py create mode 100644 antelab/runs/worker.py create mode 100644 docs/adr/0001-background-experiment-runs.md create mode 100644 docs/superpowers/specs/2026-07-13-background-experiment-runs-architecture.md create mode 100644 goals/antelab-background-experiment-runs.goal.md create mode 100644 goals/background-runs-precommit-closure.goal.md create mode 100644 goals/background-runs-readiness-repair.goal.md create mode 100644 reports/goals/antelab-background-experiment-runs/round-01.md create mode 100644 reports/goals/antelab-background-experiment-runs/round-02.md create mode 100644 reports/goals/antelab-background-experiment-runs/round-03.md create mode 100644 reports/goals/antelab-background-experiment-runs/round-04.md create mode 100644 reports/goals/background-runs-precommit-closure-round-01.md create mode 100644 reports/goals/background-runs-precommit-closure-round-02.md create mode 100644 reports/goals/background-runs-readiness-repair-round-01.md create mode 100644 reports/goals/background-runs-readiness-repair-round-02.md create mode 100644 reports/goals/background-runs-readiness-repair-round-03.md create mode 100644 scripts/validate_background_runs_readiness.py create mode 100644 state/antelab-background-experiment-runs.STATE.md create mode 100644 state/background-runs-precommit-closure.STATE.md create mode 100644 state/background-runs-readiness-repair.STATE.md create mode 100644 tests/manifests/background-run-scenarios.txt create mode 100644 tests/manifests/background-runs-round-one.sha256 create mode 100644 tests/runs/conftest.py create mode 100644 tests/runs/test_adapter_contracts.py create mode 100644 tests/runs/test_durable_adapters.py create mode 100644 tests/runs/test_experiment_runs.py create mode 100644 tests/runs/test_inprocess_engine.py create mode 100644 tests/runs/test_run_repository.py create mode 100644 tests/runs/test_run_worker.py create mode 100644 tests/test_background_runs_verifier.py diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..1ca7502 --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,35 @@ +# AnteLab Experiment Execution + +This context names the durable lifecycle around one scientifically reproducible +AnteLab experiment. + +## Language + +**Experiment Run**: +A durable, observable execution of one immutable validated experiment request. +_Avoid_: Job, task, process + +**Run Request**: +The immutable scientific inputs and version bindings that identify what an +Experiment Run must execute. +_Avoid_: Job spec, payload + +**Run Attempt**: +One actual computation of an Experiment Run. Recovery may create another Run +Attempt without creating another Experiment Run. +_Avoid_: Retry job, resumed run + +**Run Snapshot**: +An immutable revision of an Experiment Run's current lifecycle, attempt, and +progress facts. +_Avoid_: Job status, mutable run + +**Published Artifact**: +The immutable, scientifically validated result attached to a completed +Experiment Run and safe for callers to retrieve. +_Avoid_: Output file, partial result + +**Deterministic Replay**: +Recovery that starts a new Run Attempt from tick zero using the same Run +Request. It is not checkpoint resume. +_Avoid_: Resume, continue diff --git a/antelab/runs/__init__.py b/antelab/runs/__init__.py new file mode 100644 index 0000000..98d2d59 --- /dev/null +++ b/antelab/runs/__init__.py @@ -0,0 +1,27 @@ +"""Durable background Experiment Runs.""" + +from antelab.runs.artifacts import InMemoryArtifactRepository +from antelab.runs.errors import ( + ArtifactIntegrityError, + ArtifactNotReadyError, + IdempotencyConflictError, + RunNotFoundError, + RunStateConflictError, +) +from antelab.runs.manager import ExperimentRuns +from antelab.runs.model import RunId, RunRequest, RunSnapshot +from antelab.runs.repository import InMemoryRunRepository + +__all__ = [ + "ArtifactIntegrityError", + "ArtifactNotReadyError", + "ExperimentRuns", + "IdempotencyConflictError", + "InMemoryArtifactRepository", + "InMemoryRunRepository", + "RunId", + "RunNotFoundError", + "RunRequest", + "RunSnapshot", + "RunStateConflictError", +] diff --git a/antelab/runs/artifacts.py b/antelab/runs/artifacts.py new file mode 100644 index 0000000..a5465f0 --- /dev/null +++ b/antelab/runs/artifacts.py @@ -0,0 +1,272 @@ +"""ArtifactRepository Interface and in-memory Adapter.""" + +from __future__ import annotations + +import json +import os +from hashlib import sha256 +from pathlib import Path +from typing import Protocol, cast +from uuid import uuid4 + +from antelab.artifacts import RunArtifact +from antelab.runs.errors import ArtifactIntegrityError +from antelab.runs.model import ArtifactRef + + +class ArtifactRepository(Protocol): + def stage(self, artifact: RunArtifact) -> ArtifactRef: ... + + def publish(self, reference: ArtifactRef) -> ArtifactRef: ... + + def load(self, reference: ArtifactRef) -> RunArtifact: ... + + def exists(self, reference: ArtifactRef) -> bool: ... + + +class InMemoryArtifactRepository: + def __init__(self) -> None: + self._artifacts: dict[str, tuple[ArtifactRef, bytes]] = {} + self._staged: dict[str, tuple[ArtifactRef, bytes]] = {} + + def stage(self, artifact: RunArtifact) -> ArtifactRef: + artifact.validate() + payload = ( + json.dumps( + artifact.to_dict(), + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + digest = sha256(payload).hexdigest() + reference = ArtifactRef( + artifact_id=f"sha256/{digest}", + sha256=digest, + byte_size=len(payload), + schema_version=artifact.schema_version, + ) + parsed = _artifact_from_mapping( + cast(dict[str, object], json.loads(payload)) + ) + parsed.validate() + if _canonical_bytes(parsed) != payload: + raise ArtifactIntegrityError("staged artifact is not canonical") + self._staged[reference.artifact_id] = (reference, payload) + return reference + + def publish(self, reference: ArtifactRef) -> ArtifactRef: + try: + staged_ref, payload = self._staged[reference.artifact_id] + except KeyError as error: + raise ArtifactIntegrityError("staged artifact is missing") from error + if staged_ref != reference or sha256(payload).hexdigest() != reference.sha256: + raise ArtifactIntegrityError("staged artifact metadata differs") + existing = self._artifacts.get(reference.artifact_id) + if existing is not None and existing[0] != reference: + raise ArtifactIntegrityError("immutable artifact identity conflicts") + self._artifacts[reference.artifact_id] = (reference, payload) + return reference + + def load(self, reference: ArtifactRef) -> RunArtifact: + try: + stored_ref, payload = self._artifacts[reference.artifact_id] + except KeyError as error: + raise ArtifactIntegrityError("published artifact is missing") from error + if stored_ref != reference: + raise ArtifactIntegrityError("published artifact metadata differs") + if sha256(payload).hexdigest() != reference.sha256: + raise ArtifactIntegrityError("published artifact digest differs") + try: + artifact = _artifact_from_mapping( + cast(dict[str, object], json.loads(payload)) + ) + artifact.validate() + except (KeyError, TypeError, ValueError) as error: + raise ArtifactIntegrityError("published artifact is invalid") from error + if _canonical_bytes(artifact) != payload: + raise ArtifactIntegrityError("published artifact is not canonical") + return artifact + + def exists(self, reference: ArtifactRef) -> bool: + return reference.artifact_id in self._artifacts + + +class FilesystemArtifactRepository: + def __init__(self, root: Path) -> None: + self._root = root.resolve() + self._staging = self._root / ".staging" + self._published = self._root / "sha256" + self._staged: dict[str, Path] = {} + + def stage(self, artifact: RunArtifact) -> ArtifactRef: + artifact.validate() + payload = _canonical_bytes(artifact) + parsed = _artifact_from_mapping( + cast(dict[str, object], json.loads(payload)) + ) + parsed.validate() + if _canonical_bytes(parsed) != payload: + raise ArtifactIntegrityError("staged artifact is not canonical") + digest = sha256(payload).hexdigest() + reference = ArtifactRef( + artifact_id=f"sha256/{digest}", + sha256=digest, + byte_size=len(payload), + schema_version=artifact.schema_version, + ) + self._staging.mkdir(parents=True, exist_ok=True) + path = self._staging / f"{digest}.{uuid4().hex}.tmp" + with path.open("xb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + self._staged[reference.artifact_id] = path + return reference + + def publish(self, reference: ArtifactRef) -> ArtifactRef: + final = self._path(reference) + if final.exists(): + self._verify_bytes(final.read_bytes(), reference) + return reference + staged = self._staged.get(reference.artifact_id) + if staged is None or not staged.exists(): + candidates = sorted(self._staging.glob(f"{reference.sha256}.*.tmp")) + if not candidates: + raise ArtifactIntegrityError("staged artifact is missing") + staged = candidates[0] + payload = staged.read_bytes() + self._verify_bytes(payload, reference) + final.parent.mkdir(parents=True, exist_ok=True) + try: + final.hardlink_to(staged) + except FileExistsError: + self._verify_bytes(final.read_bytes(), reference) + directory_fd = os.open(final.parent, os.O_RDONLY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + return reference + + def exists(self, reference: ArtifactRef) -> bool: + final = self._path(reference) + if not final.exists(): + return False + self._verify_bytes(final.read_bytes(), reference) + return True + + def load(self, reference: ArtifactRef) -> RunArtifact: + path = self._path(reference) + if not path.exists(): + raise ArtifactIntegrityError("published artifact is missing") + payload = path.read_bytes() + self._verify_bytes(payload, reference) + try: + raw = json.loads(payload) + artifact = _artifact_from_mapping(cast(dict[str, object], raw)) + artifact.validate() + except (KeyError, TypeError, ValueError) as error: + raise ArtifactIntegrityError("published artifact is invalid") from error + if _canonical_bytes(artifact) != payload: + raise ArtifactIntegrityError("published artifact is not canonical") + return artifact + + def reference_for(self, artifact_id: str) -> ArtifactRef: + if not artifact_id.startswith("sha256/"): + raise ArtifactIntegrityError("artifact identity is not content addressed") + digest = artifact_id.removeprefix("sha256/") + path = (self._root / artifact_id).resolve() + if not path.is_relative_to(self._root): + raise ArtifactIntegrityError("artifact path escapes managed root") + if not path.exists(): + raise ArtifactIntegrityError("published artifact is missing") + payload = path.read_bytes() + reference = ArtifactRef( + artifact_id=artifact_id, + sha256=digest, + byte_size=len(payload), + schema_version=1, + ) + self._verify_bytes(payload, reference) + return reference + + def recoverable_reference_for(self, artifact_id: str) -> ArtifactRef: + if not artifact_id.startswith("sha256/"): + raise ArtifactIntegrityError("artifact identity is not content addressed") + digest = artifact_id.removeprefix("sha256/") + if len(digest) != 64: + raise ArtifactIntegrityError("artifact digest is invalid") + final = (self._root / artifact_id).resolve() + if not final.is_relative_to(self._root): + raise ArtifactIntegrityError("artifact path escapes managed root") + if final.exists(): + return self.reference_for(artifact_id) + candidates = sorted(self._staging.glob(f"{digest}.*.tmp")) + if not candidates: + raise ArtifactIntegrityError("staged artifact is missing") + payload = candidates[0].read_bytes() + reference = ArtifactRef( + artifact_id=artifact_id, + sha256=digest, + byte_size=len(payload), + schema_version=1, + ) + self._verify_bytes(payload, reference) + return reference + + def _path(self, reference: ArtifactRef) -> Path: + if reference.artifact_id != f"sha256/{reference.sha256}": + raise ArtifactIntegrityError("artifact identity is not content addressed") + path = (self._root / reference.artifact_id).resolve() + if not path.is_relative_to(self._root): + raise ArtifactIntegrityError("artifact path escapes managed root") + return path + + @staticmethod + def _verify_bytes(payload: bytes, reference: ArtifactRef) -> None: + if len(payload) != reference.byte_size: + raise ArtifactIntegrityError("artifact byte size differs") + if sha256(payload).hexdigest() != reference.sha256: + raise ArtifactIntegrityError("artifact digest differs") + + +def _canonical_bytes(artifact: RunArtifact) -> bytes: + return ( + json.dumps( + artifact.to_dict(), + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + + +def _artifact_from_mapping(raw: dict[str, object]) -> RunArtifact: + return RunArtifact( + schema_version=cast(int, raw["schema_version"]), + engine_version=cast(str, raw["engine_version"]), + run_id=cast(str, raw["run_id"]), + condition=cast(str, raw["condition"]), + seed=cast(int, raw["seed"]), + ticks_requested=cast(int, raw["ticks_requested"]), + ticks_completed=cast(int, raw["ticks_completed"]), + status=cast(str, raw["status"]), + config=cast(dict[str, object], raw["config"]), + ancestor_genome=cast(dict[str, object], raw["ancestor_genome"]), + summary=cast(dict[str, int], raw["summary"]), + metrics=tuple(cast(list[dict[str, int | str]], raw["metrics"])), + lineage=tuple(cast(list[dict[str, int]], raw["lineage"])), + genome_distribution=tuple( + cast(list[dict[str, object]], raw["genome_distribution"]) + ), + notable_events=tuple( + cast(list[dict[str, int | str]], raw["notable_events"]) + ), + final_state=cast(dict[str, object], raw["final_state"]), + final_checksum=cast(str, raw["final_checksum"]), + ) diff --git a/antelab/runs/errors.py b/antelab/runs/errors.py new file mode 100644 index 0000000..e48b87c --- /dev/null +++ b/antelab/runs/errors.py @@ -0,0 +1,25 @@ +"""Stable public failures for Experiment Runs.""" + + +class RunError(RuntimeError): + pass + + +class IdempotencyConflictError(RunError): + pass + + +class RunNotFoundError(RunError): + pass + + +class RunStateConflictError(RunError): + pass + + +class ArtifactNotReadyError(RunError): + pass + + +class ArtifactIntegrityError(RunError): + pass diff --git a/antelab/runs/manager.py b/antelab/runs/manager.py new file mode 100644 index 0000000..2bcc60e --- /dev/null +++ b/antelab/runs/manager.py @@ -0,0 +1,52 @@ +"""Caller-facing ExperimentRuns deep Module.""" + +from __future__ import annotations + +from collections.abc import Callable +from uuid import uuid4 + +from antelab.artifacts import RunArtifact +from antelab.runs.artifacts import ArtifactRepository +from antelab.runs.errors import ArtifactNotReadyError +from antelab.runs.model import IdempotencyKey, RunId, RunRequest, RunSnapshot +from antelab.runs.repository import RunRepository + + +class ExperimentRuns: + def __init__( + self, + repository: RunRepository, + artifacts: ArtifactRepository, + *, + id_generator: Callable[[], str] = lambda: uuid4().hex, + ) -> None: + self._repository = repository + self._artifacts = artifacts + self._id_generator = id_generator + + def submit( + self, request: RunRequest, *, idempotency_key: str + ) -> RunSnapshot: + if not isinstance(idempotency_key, str) or not idempotency_key.strip(): + raise ValueError("idempotency_key must be a non-empty string") + if len(idempotency_key) > 256: + raise ValueError("idempotency_key must be at most 256 characters") + return self._repository.create_idempotent( + request, + IdempotencyKey(idempotency_key), + lambda: RunId(self._id_generator()), + ) + + def get(self, run_id: RunId) -> RunSnapshot: + return self._repository.get(run_id) + + def cancel(self, run_id: RunId) -> RunSnapshot: + return self._repository.cancel(run_id) + + def artifact(self, run_id: RunId) -> RunArtifact: + snapshot = self._repository.get(run_id) + if snapshot.state != "completed" or snapshot.artifact is None: + raise ArtifactNotReadyError( + "artifact is available only for completed runs" + ) + return self._artifacts.load(snapshot.artifact) diff --git a/antelab/runs/model.py b/antelab/runs/model.py new file mode 100644 index 0000000..706efa6 --- /dev/null +++ b/antelab/runs/model.py @@ -0,0 +1,162 @@ +"""Domain model for durable background Experiment Runs.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import UTC, datetime +from hashlib import sha256 +from typing import Literal, NewType + +from antelab import __version__ +from antelab.core.config import SimulationConfig + +RunId = NewType("RunId", str) +IdempotencyKey = NewType("IdempotencyKey", str) +RunState = Literal[ + "queued", "running", "cancelling", "completed", "failed", "cancelled" +] +RunPhase = Literal["waiting", "computing", "verifying", "publishing", "terminal"] + + +@dataclass(frozen=True) +class RunRequest: + config: SimulationConfig + + def __post_init__(self) -> None: + self.config.validate() + + def digest(self) -> str: + payload = { + "artifact_schema_version": 1, + "config": self.config.to_dict(), + "engine_version": __version__, + } + encoded = json.dumps( + payload, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode("utf-8") + return sha256(encoded).hexdigest() + + +@dataclass(frozen=True) +class RunProgress: + attempt: int + attempt_tick: int + max_tick_seen: int + ticks_requested: int + updated_at: datetime + + def __post_init__(self) -> None: + values = (self.attempt, self.attempt_tick, self.max_tick_seen, self.ticks_requested) + if any(type(value) is not int for value in values): + raise ValueError("run progress values must be exact integers") + if ( + self.attempt < 0 + or self.ticks_requested < 1 + or not 0 <= self.attempt_tick <= self.ticks_requested + or self.max_tick_seen < self.attempt_tick + ): + raise ValueError("run progress values are inconsistent") + if self.updated_at.tzinfo is None: + raise ValueError("run progress timestamp must be timezone-aware") + + +@dataclass(frozen=True) +class ArtifactRef: + artifact_id: str + sha256: str + byte_size: int + schema_version: int + + def __post_init__(self) -> None: + if not self.artifact_id or len(self.sha256) != 64 or self.byte_size <= 0: + raise ValueError("artifact reference is invalid") + if self.schema_version != 1: + raise ValueError("artifact schema version must equal 1") + + +@dataclass(frozen=True) +class PublicFailure: + code: str + message: str + retryable: bool + attempt: int + + def __post_init__(self) -> None: + if not self.code or not self.message or self.attempt < 1: + raise ValueError("public failure is invalid") + if len(self.message) > 256: + raise ValueError("public failure message is too long") + + +@dataclass(frozen=True) +class RunSnapshot: + run_id: RunId + revision: int + request_digest: str + state: RunState + phase: RunPhase + progress: RunProgress + artifact: ArtifactRef | None = None + failure: PublicFailure | None = None + + def __post_init__(self) -> None: + expected_phase = { + "queued": {"waiting"}, + "running": {"computing", "verifying", "publishing"}, + "cancelling": {"computing"}, + "completed": {"terminal"}, + "failed": {"terminal"}, + "cancelled": {"terminal"}, + } + if self.phase not in expected_phase[self.state]: + raise ValueError("run state and phase are inconsistent") + if (self.artifact is not None) != (self.state == "completed"): + raise ValueError("only completed runs have an artifact") + if (self.failure is not None) != (self.state == "failed"): + raise ValueError("only failed runs have a failure") + if self.revision < 0: + raise ValueError("run revision must be non-negative") + + +@dataclass(frozen=True) +class Lease: + run_id: RunId + attempt: int + fencing_token: str + expires_at: datetime + revision: int + + def __post_init__(self) -> None: + if self.attempt < 1 or not self.fencing_token or self.revision < 1: + raise ValueError("worker lease is invalid") + if self.expires_at.tzinfo is None: + raise ValueError("worker lease expiry must be timezone-aware") + + +@dataclass +class _RunRecord: + snapshot: RunSnapshot + request: RunRequest + idempotency_key: IdempotencyKey + lease: Lease | None = None + recovery_count: int = 0 + candidate_artifact_id: str | None = None + + +def initial_snapshot(run_id: RunId, request: RunRequest) -> RunSnapshot: + now = datetime.now(UTC) + return RunSnapshot( + run_id=run_id, + revision=0, + request_digest=request.digest(), + state="queued", + phase="waiting", + progress=RunProgress( + attempt=0, + attempt_tick=0, + max_tick_seen=0, + ticks_requested=request.config.ticks, + updated_at=now, + ), + ) diff --git a/antelab/runs/ownership.py b/antelab/runs/ownership.py new file mode 100644 index 0000000..a32b189 --- /dev/null +++ b/antelab/runs/ownership.py @@ -0,0 +1,14 @@ +"""Scientific ownership checks shared by normal and recovery publication.""" + +from antelab.artifacts import RunArtifact +from antelab.runs.model import RunRequest + + +def validate_artifact_ownership(request: RunRequest, artifact: RunArtifact) -> None: + artifact.validate() + if ( + artifact.config != request.config.to_dict() + or artifact.ticks_requested != request.config.ticks + or artifact.seed != request.config.seed + ): + raise RuntimeError("published artifact does not belong to the run request") diff --git a/antelab/runs/recovery.py b/antelab/runs/recovery.py new file mode 100644 index 0000000..2553fcd --- /dev/null +++ b/antelab/runs/recovery.py @@ -0,0 +1,38 @@ +"""Crash reconciliation for already-published immutable artifacts.""" + +from collections.abc import Callable +from datetime import UTC, datetime + +from antelab.runs.artifacts import FilesystemArtifactRepository +from antelab.runs.model import RunId +from antelab.runs.ownership import validate_artifact_ownership +from antelab.runs.sqlite_repository import SqliteRunRepository + + +def reconcile_publications( + runs: SqliteRunRepository, + artifacts: FilesystemArtifactRepository, + *, + now: Callable[[], datetime] = lambda: datetime.now(UTC), +) -> tuple[RunId, ...]: + completed: list[RunId] = [] + for lease, artifact_id in runs.publishing_candidates(): + try: + reference = artifacts.recoverable_reference_for(artifact_id) + artifacts.publish(reference) + artifact = artifacts.load(reference) + request = runs.request(lease.run_id) + validate_artifact_ownership(request, artifact) + except OSError: + raise + except (RuntimeError, ValueError): + runs.fail_reconciled( + lease, + code="artifact_recovery_failed", + message="artifact publication recovery failed", + now=now(), + ) + continue + runs.finalize_reconciled(lease, artifact=reference, now=now()) + completed.append(lease.run_id) + return tuple(completed) diff --git a/antelab/runs/repository.py b/antelab/runs/repository.py new file mode 100644 index 0000000..114177c --- /dev/null +++ b/antelab/runs/repository.py @@ -0,0 +1,464 @@ +"""RunRepository Interface and in-memory Adapter.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from threading import RLock +from typing import Protocol + +from antelab.runs.errors import ( + IdempotencyConflictError, + RunNotFoundError, + RunStateConflictError, +) +from antelab.runs.model import ( + ArtifactRef, + IdempotencyKey, + Lease, + PublicFailure, + RunId, + RunPhase, + RunRequest, + RunSnapshot, + RunState, + _RunRecord, + initial_snapshot, +) + + +class RunRepository(Protocol): + def create_idempotent( + self, + request: RunRequest, + idempotency_key: IdempotencyKey, + run_id_factory: Callable[[], RunId], + ) -> RunSnapshot: ... + + def get(self, run_id: RunId) -> RunSnapshot: ... + + def request(self, run_id: RunId) -> RunRequest: ... + + def cancel(self, run_id: RunId) -> RunSnapshot: ... + + def claim_next( + self, + *, + now: datetime, + lease_duration: timedelta, + token_factory: Callable[[], str], + ) -> Lease | None: ... + + def renew_lease( + self, + lease: Lease, + *, + now: datetime, + lease_duration: timedelta, + ) -> Lease: ... + + def record_progress( + self, lease: Lease, *, tick: int, now: datetime + ) -> RunSnapshot: ... + + def begin_verifying(self, lease: Lease, *, now: datetime) -> RunSnapshot: ... + + def begin_publishing( + self, lease: Lease, *, artifact_id: str, now: datetime + ) -> RunSnapshot: ... + + def acknowledge_cancel(self, lease: Lease, *, now: datetime) -> RunSnapshot: ... + + def complete( + self, lease: Lease, *, artifact: ArtifactRef, now: datetime + ) -> RunSnapshot: ... + + def fail( + self, + lease: Lease, + *, + code: str, + message: str, + retryable: bool, + now: datetime, + ) -> RunSnapshot: ... + + def fail_reconciled( + self, lease: Lease, *, code: str, message: str, now: datetime + ) -> RunSnapshot: ... + + +class InMemoryRunRepository: + def __init__(self) -> None: + self._records: dict[RunId, _RunRecord] = {} + self._idempotency: dict[IdempotencyKey, RunId] = {} + self._lock = RLock() + + def create_idempotent( + self, + request: RunRequest, + idempotency_key: IdempotencyKey, + run_id_factory: Callable[[], RunId], + ) -> RunSnapshot: + with self._lock: + existing_id = self._idempotency.get(idempotency_key) + if existing_id is not None: + existing = self._records[existing_id] + if existing.snapshot.request_digest != request.digest(): + raise IdempotencyConflictError( + "idempotency key is already bound to another request" + ) + return existing.snapshot + run_id = run_id_factory() + snapshot = initial_snapshot(run_id, request) + self._records[run_id] = _RunRecord( + snapshot=snapshot, + request=request, + idempotency_key=idempotency_key, + ) + self._idempotency[idempotency_key] = run_id + return snapshot + + def get(self, run_id: RunId) -> RunSnapshot: + with self._lock: + return self._record(run_id).snapshot + + def request(self, run_id: RunId) -> RunRequest: + with self._lock: + return self._record(run_id).request + + def cancel(self, run_id: RunId) -> RunSnapshot: + with self._lock: + record = self._record(run_id) + current = record.snapshot + if current.state in {"completed", "failed", "cancelled", "cancelling"}: + return current + if current.state == "running" and current.phase != "computing": + return current + state: RunState = ( + "cancelled" if current.state == "queued" else "cancelling" + ) + phase: RunPhase = "terminal" if state == "cancelled" else current.phase + record.snapshot = replace( + current, + revision=current.revision + 1, + state=state, + phase=phase, + progress=replace( + current.progress, + updated_at=datetime.now(UTC), + ), + ) + if record.lease is not None: + record.lease = replace( + record.lease, revision=record.snapshot.revision + ) + return record.snapshot + + def claim_next( + self, + *, + now: datetime, + lease_duration: timedelta, + token_factory: Callable[[], str], + ) -> Lease | None: + with self._lock: + for run_id, record in self._records.items(): + if record.snapshot.state != "queued": + continue + current = record.snapshot + attempt = current.progress.attempt + 1 + next_snapshot = replace( + current, + revision=current.revision + 1, + state="running", + phase="computing", + progress=replace( + current.progress, + attempt=attempt, + attempt_tick=0, + updated_at=now, + ), + ) + lease = Lease( + run_id=run_id, + attempt=attempt, + fencing_token=token_factory(), + expires_at=now + lease_duration, + revision=next_snapshot.revision, + ) + record.snapshot = next_snapshot + record.lease = lease + return lease + return None + + def record_progress( + self, lease: Lease, *, tick: int, now: datetime + ) -> RunSnapshot: + with self._lock: + record = self._leased_record(lease, now=now) + current = record.snapshot + if current.state != "running" or current.phase != "computing": + raise RunStateConflictError("run is not accepting progress") + if tick < current.progress.attempt_tick: + raise RunStateConflictError("attempt progress cannot move backwards") + if tick > current.progress.ticks_requested: + raise RunStateConflictError("attempt progress exceeds requested ticks") + record.snapshot = replace( + current, + revision=current.revision + 1, + progress=replace( + current.progress, + attempt_tick=tick, + max_tick_seen=max(current.progress.max_tick_seen, tick), + updated_at=now, + ), + ) + record.lease = replace(lease, revision=record.snapshot.revision) + return record.snapshot + + def renew_lease( + self, + lease: Lease, + *, + now: datetime, + lease_duration: timedelta, + ) -> Lease: + with self._lock: + record = self._leased_record(lease, now=now) + renewed = replace(lease, expires_at=now + lease_duration) + record.lease = renewed + return renewed + + def begin_verifying(self, lease: Lease, *, now: datetime) -> RunSnapshot: + with self._lock: + record = self._leased_record(lease, now=now) + current = record.snapshot + if current.state != "running" or current.phase != "computing": + raise RunStateConflictError("run cannot enter verification") + record.snapshot = replace( + current, + revision=current.revision + 1, + phase="verifying", + progress=replace(current.progress, updated_at=now), + ) + record.lease = replace(lease, revision=record.snapshot.revision) + return record.snapshot + + def begin_publishing( + self, lease: Lease, *, artifact_id: str, now: datetime + ) -> RunSnapshot: + with self._lock: + record = self._leased_record(lease, now=now) + current = record.snapshot + if current.state != "running" or current.phase != "verifying": + raise RunStateConflictError("run cannot enter publication") + record.candidate_artifact_id = artifact_id + record.snapshot = replace( + current, + revision=current.revision + 1, + phase="publishing", + progress=replace(current.progress, updated_at=now), + ) + record.lease = replace(lease, revision=record.snapshot.revision) + return record.snapshot + + def acknowledge_cancel(self, lease: Lease, *, now: datetime) -> RunSnapshot: + with self._lock: + record = self._leased_record(lease, now=now) + current = record.snapshot + if current.state != "cancelling": + raise RunStateConflictError("run has no cancellation to acknowledge") + record.snapshot = replace( + current, + revision=current.revision + 1, + state="cancelled", + phase="terminal", + progress=replace(current.progress, updated_at=now), + ) + record.lease = None + return record.snapshot + + def complete( + self, lease: Lease, *, artifact: ArtifactRef, now: datetime + ) -> RunSnapshot: + with self._lock: + record = self._leased_record(lease, now=now) + current = record.snapshot + if current.state != "running" or current.phase != "publishing": + raise RunStateConflictError("run cannot complete from current state") + if record.candidate_artifact_id != artifact.artifact_id: + raise RunStateConflictError("published artifact differs from candidate") + record.snapshot = replace( + current, + revision=current.revision + 1, + state="completed", + phase="terminal", + artifact=artifact, + progress=replace(current.progress, updated_at=now), + ) + record.lease = None + return record.snapshot + + def finalize_reconciled( + self, lease: Lease, *, artifact: ArtifactRef, now: datetime + ) -> RunSnapshot: + with self._lock: + record = self._leased_record(lease) + current = record.snapshot + if current.state != "running" or current.phase != "publishing": + raise RunStateConflictError("run is not awaiting publication recovery") + if record.candidate_artifact_id != artifact.artifact_id: + raise RunStateConflictError("recovered artifact differs from candidate") + record.snapshot = replace( + current, + revision=current.revision + 1, + state="completed", + phase="terminal", + artifact=artifact, + progress=replace(current.progress, updated_at=now), + ) + record.lease = None + return record.snapshot + + def fail_reconciled( + self, lease: Lease, *, code: str, message: str, now: datetime + ) -> RunSnapshot: + with self._lock: + record = self._leased_record(lease) + current = record.snapshot + if current.state != "running" or current.phase != "publishing": + raise RunStateConflictError("run is not awaiting publication recovery") + record.snapshot = replace( + current, + revision=current.revision + 1, + state="failed", + phase="terminal", + failure=PublicFailure( + code=code, + message=message[:256], + retryable=False, + attempt=current.progress.attempt, + ), + progress=replace(current.progress, updated_at=now), + ) + record.lease = None + return record.snapshot + + def fail( + self, + lease: Lease, + *, + code: str, + message: str, + retryable: bool, + now: datetime, + ) -> RunSnapshot: + with self._lock: + record = self._leased_record(lease, now=now) + current = record.snapshot + if current.state != "running": + raise RunStateConflictError("run cannot fail from current state") + record.snapshot = replace( + current, + revision=current.revision + 1, + state="failed", + phase="terminal", + failure=PublicFailure( + code=code, + message=message[:256], + retryable=retryable, + attempt=current.progress.attempt, + ), + progress=replace(current.progress, updated_at=now), + ) + record.lease = None + return record.snapshot + + def recover_expired(self, *, now: datetime) -> tuple[RunSnapshot, ...]: + recovered: list[RunSnapshot] = [] + with self._lock: + for record in self._records.values(): + lease = record.lease + if lease is None or lease.expires_at >= now: + continue + current = record.snapshot + if current.state == "cancelling": + next_snapshot = replace( + current, + revision=current.revision + 1, + state="cancelled", + phase="terminal", + progress=replace(current.progress, updated_at=now), + ) + elif current.state == "running": + if current.phase == "publishing": + continue + record.recovery_count += 1 + if record.recovery_count >= 3: + next_snapshot = replace( + current, + revision=current.revision + 1, + state="failed", + phase="terminal", + failure=PublicFailure( + code="worker_recovery_exhausted", + message="worker recovery attempts were exhausted", + retryable=False, + attempt=current.progress.attempt, + ), + progress=replace(current.progress, updated_at=now), + ) + else: + next_snapshot = replace( + current, + revision=current.revision + 1, + state="queued", + phase="waiting", + progress=replace( + current.progress, + attempt_tick=0, + updated_at=now, + ), + ) + else: + continue + record.snapshot = next_snapshot + record.lease = None + recovered.append(next_snapshot) + return tuple(recovered) + + def publishing_candidates(self) -> tuple[tuple[Lease, str], ...]: + with self._lock: + return tuple( + (record.lease, record.candidate_artifact_id) + for record in self._records.values() + if record.snapshot.state == "running" + and record.snapshot.phase == "publishing" + and record.lease is not None + and record.candidate_artifact_id is not None + ) + + def _leased_record( + self, lease: Lease, *, now: datetime | None = None + ) -> _RunRecord: + record = self._record(lease.run_id) + current = record.lease + if ( + current is None + or current.attempt != lease.attempt + or current.fencing_token != lease.fencing_token + ): + raise RunStateConflictError("worker lease is stale") + if lease.revision != record.snapshot.revision: + raise RunStateConflictError("worker expected revision is stale") + if now is not None and current.expires_at < now: + raise RunStateConflictError("worker lease is expired") + return record + + def _record(self, run_id: RunId) -> _RunRecord: + try: + return self._records[run_id] + except KeyError as error: + raise RunNotFoundError(f"unknown run: {run_id}") from error diff --git a/antelab/runs/sqlite_repository.py b/antelab/runs/sqlite_repository.py new file mode 100644 index 0000000..348da4e --- /dev/null +++ b/antelab/runs/sqlite_repository.py @@ -0,0 +1,356 @@ +"""SQLite durable Adapter for the RunRepository contract.""" + +from __future__ import annotations + +import json +import sqlite3 +from collections.abc import Callable +from datetime import datetime, timedelta +from pathlib import Path +from threading import RLock +from typing import TypeVar, cast + +from antelab.core.config import SimulationConfig +from antelab.runs.model import ( + ArtifactRef, + IdempotencyKey, + Lease, + PublicFailure, + RunId, + RunPhase, + RunProgress, + RunRequest, + RunSnapshot, + RunState, + _RunRecord, +) +from antelab.runs.repository import InMemoryRunRepository + +Result = TypeVar("Result") + + +class SqliteRunRepository(InMemoryRunRepository): + def __init__(self, path: Path) -> None: + super().__init__() + self._path = path + self._sqlite_lock = RLock() + path.parent.mkdir(parents=True, exist_ok=True) + with self._connect() as connection: + connection.execute( + "CREATE TABLE IF NOT EXISTS run_state " + "(singleton INTEGER PRIMARY KEY CHECK(singleton = 1), payload TEXT NOT NULL)" + ) + connection.execute( + "INSERT OR IGNORE INTO run_state(singleton, payload) " + "VALUES (1, '{\"schema_version\":1,\"runs\":{}}')" + ) + self._read(lambda: None) + + def create_idempotent( + self, + request: RunRequest, + idempotency_key: IdempotencyKey, + run_id_factory: Callable[[], RunId], + ) -> RunSnapshot: + return self._write( + lambda: super(SqliteRunRepository, self).create_idempotent( + request, idempotency_key, run_id_factory + ) + ) + + def get(self, run_id: RunId) -> RunSnapshot: + return self._read(lambda: super(SqliteRunRepository, self).get(run_id)) + + def request(self, run_id: RunId) -> RunRequest: + return self._read(lambda: super(SqliteRunRepository, self).request(run_id)) + + def cancel(self, run_id: RunId) -> RunSnapshot: + return self._write(lambda: super(SqliteRunRepository, self).cancel(run_id)) + + def claim_next( + self, + *, + now: datetime, + lease_duration: timedelta, + token_factory: Callable[[], str], + ) -> Lease | None: + return self._write( + lambda: super(SqliteRunRepository, self).claim_next( + now=now, + lease_duration=lease_duration, + token_factory=token_factory, + ) + ) + + def record_progress( + self, lease: Lease, *, tick: int, now: datetime + ) -> RunSnapshot: + return self._write( + lambda: super(SqliteRunRepository, self).record_progress( + lease, tick=tick, now=now + ) + ) + + def renew_lease( + self, + lease: Lease, + *, + now: datetime, + lease_duration: timedelta, + ) -> Lease: + return self._write( + lambda: super(SqliteRunRepository, self).renew_lease( + lease, now=now, lease_duration=lease_duration + ) + ) + + def begin_verifying(self, lease: Lease, *, now: datetime) -> RunSnapshot: + return self._write( + lambda: super(SqliteRunRepository, self).begin_verifying(lease, now=now) + ) + + def begin_publishing( + self, lease: Lease, *, artifact_id: str, now: datetime + ) -> RunSnapshot: + return self._write( + lambda: super(SqliteRunRepository, self).begin_publishing( + lease, artifact_id=artifact_id, now=now + ) + ) + + def acknowledge_cancel(self, lease: Lease, *, now: datetime) -> RunSnapshot: + return self._write( + lambda: super(SqliteRunRepository, self).acknowledge_cancel( + lease, now=now + ) + ) + + def complete( + self, lease: Lease, *, artifact: ArtifactRef, now: datetime + ) -> RunSnapshot: + return self._write( + lambda: super(SqliteRunRepository, self).complete( + lease, artifact=artifact, now=now + ) + ) + + def finalize_reconciled( + self, lease: Lease, *, artifact: ArtifactRef, now: datetime + ) -> RunSnapshot: + return self._write( + lambda: super(SqliteRunRepository, self).finalize_reconciled( + lease, artifact=artifact, now=now + ) + ) + + def fail( + self, + lease: Lease, + *, + code: str, + message: str, + retryable: bool, + now: datetime, + ) -> RunSnapshot: + return self._write( + lambda: super(SqliteRunRepository, self).fail( + lease, + code=code, + message=message, + retryable=retryable, + now=now, + ) + ) + + def fail_reconciled( + self, lease: Lease, *, code: str, message: str, now: datetime + ) -> RunSnapshot: + return self._write( + lambda: super(SqliteRunRepository, self).fail_reconciled( + lease, code=code, message=message, now=now + ) + ) + + def recover_expired(self, *, now: datetime) -> tuple[RunSnapshot, ...]: + return self._write( + lambda: super(SqliteRunRepository, self).recover_expired(now=now) + ) + + def publishing_candidates(self) -> tuple[tuple[Lease, str], ...]: + return self._read( + lambda: super(SqliteRunRepository, self).publishing_candidates() + ) + + def _connect(self) -> sqlite3.Connection: + connection = sqlite3.connect(self._path, timeout=10) + connection.execute("PRAGMA journal_mode=WAL") + return connection + + def _read(self, operation: Callable[[], Result]) -> Result: + with self._sqlite_lock, self._connect() as connection: + self._load(connection) + return operation() + + def _write(self, operation: Callable[[], Result]) -> Result: + with self._sqlite_lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + self._load(connection) + result = operation() + connection.execute( + "UPDATE run_state SET payload = ? WHERE singleton = 1", + (self._dump(),), + ) + connection.commit() + return result + + def _load(self, connection: sqlite3.Connection) -> None: + row = connection.execute( + "SELECT payload FROM run_state WHERE singleton = 1" + ).fetchone() + envelope = cast(dict[str, object], json.loads(cast(str, row[0]))) + if envelope.get("schema_version") != 1 or set(envelope) != { + "schema_version", + "runs", + }: + raise ValueError("unsupported durable run-state schema") + raw = cast(dict[str, object], envelope["runs"]) + records: dict[RunId, _RunRecord] = {} + idempotency: dict[IdempotencyKey, RunId] = {} + for run_id_raw, record_raw_object in raw.items(): + record_raw = cast(dict[str, object], record_raw_object) + run_id = RunId(run_id_raw) + request = RunRequest( + SimulationConfig.from_mapping( + cast(dict[str, object], record_raw["config"]) + ) + ) + key = IdempotencyKey(cast(str, record_raw["idempotency_key"])) + record = _RunRecord( + snapshot=_snapshot_from_dict( + cast(dict[str, object], record_raw["snapshot"]) + ), + request=request, + idempotency_key=key, + lease=_lease_from_dict( + cast(dict[str, object] | None, record_raw.get("lease")) + ), + recovery_count=cast(int, record_raw["recovery_count"]), + candidate_artifact_id=cast( + str | None, record_raw.get("candidate_artifact_id") + ), + ) + records[run_id] = record + idempotency[key] = run_id + self._records = records + self._idempotency = idempotency + + def _dump(self) -> str: + raw = { + str(run_id): { + "candidate_artifact_id": record.candidate_artifact_id, + "config": record.request.config.to_dict(), + "idempotency_key": str(record.idempotency_key), + "lease": _lease_to_dict(record.lease), + "recovery_count": record.recovery_count, + "snapshot": _snapshot_to_dict(record.snapshot), + } + for run_id, record in self._records.items() + } + return json.dumps( + {"schema_version": 1, "runs": raw}, + sort_keys=True, + separators=(",", ":"), + ) + + +def _snapshot_to_dict(snapshot: RunSnapshot) -> dict[str, object]: + return { + "artifact": None + if snapshot.artifact is None + else { + "artifact_id": snapshot.artifact.artifact_id, + "sha256": snapshot.artifact.sha256, + "byte_size": snapshot.artifact.byte_size, + "schema_version": snapshot.artifact.schema_version, + }, + "failure": None + if snapshot.failure is None + else { + "code": snapshot.failure.code, + "message": snapshot.failure.message, + "retryable": snapshot.failure.retryable, + "attempt": snapshot.failure.attempt, + }, + "phase": snapshot.phase, + "progress": { + "attempt": snapshot.progress.attempt, + "attempt_tick": snapshot.progress.attempt_tick, + "max_tick_seen": snapshot.progress.max_tick_seen, + "ticks_requested": snapshot.progress.ticks_requested, + "updated_at": snapshot.progress.updated_at.isoformat(), + }, + "request_digest": snapshot.request_digest, + "revision": snapshot.revision, + "run_id": str(snapshot.run_id), + "state": snapshot.state, + } + + +def _snapshot_from_dict(raw: dict[str, object]) -> RunSnapshot: + progress = cast(dict[str, object], raw["progress"]) + artifact_raw = cast(dict[str, object] | None, raw.get("artifact")) + failure_raw = cast(dict[str, object] | None, raw.get("failure")) + return RunSnapshot( + run_id=RunId(cast(str, raw["run_id"])), + revision=cast(int, raw["revision"]), + request_digest=cast(str, raw["request_digest"]), + state=cast(RunState, raw["state"]), + phase=cast(RunPhase, raw["phase"]), + progress=RunProgress( + attempt=cast(int, progress["attempt"]), + attempt_tick=cast(int, progress["attempt_tick"]), + max_tick_seen=cast(int, progress["max_tick_seen"]), + ticks_requested=cast(int, progress["ticks_requested"]), + updated_at=datetime.fromisoformat(cast(str, progress["updated_at"])), + ), + artifact=None + if artifact_raw is None + else ArtifactRef( + artifact_id=cast(str, artifact_raw["artifact_id"]), + sha256=cast(str, artifact_raw["sha256"]), + byte_size=cast(int, artifact_raw["byte_size"]), + schema_version=cast(int, artifact_raw["schema_version"]), + ), + failure=None + if failure_raw is None + else PublicFailure( + code=cast(str, failure_raw["code"]), + message=cast(str, failure_raw["message"]), + retryable=cast(bool, failure_raw["retryable"]), + attempt=cast(int, failure_raw["attempt"]), + ), + ) + + +def _lease_to_dict(lease: Lease | None) -> dict[str, object] | None: + if lease is None: + return None + return { + "attempt": lease.attempt, + "expires_at": lease.expires_at.isoformat(), + "fencing_token": lease.fencing_token, + "revision": lease.revision, + "run_id": str(lease.run_id), + } + + +def _lease_from_dict(raw: dict[str, object] | None) -> Lease | None: + if raw is None: + return None + return Lease( + run_id=RunId(cast(str, raw["run_id"])), + attempt=cast(int, raw["attempt"]), + fencing_token=cast(str, raw["fencing_token"]), + expires_at=datetime.fromisoformat(cast(str, raw["expires_at"])), + revision=cast(int, raw["revision"]), + ) diff --git a/antelab/runs/worker.py b/antelab/runs/worker.py new file mode 100644 index 0000000..1169f39 --- /dev/null +++ b/antelab/runs/worker.py @@ -0,0 +1,240 @@ +"""Single-concurrency RunWorker and deterministic fake engine Adapter.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from dataclasses import replace +from datetime import datetime, timedelta +from typing import Protocol + +from antelab.artifacts import RunArtifact +from antelab.core.simulation import Simulation +from antelab.experiments.runner import _metric +from antelab.runs.artifacts import ArtifactRepository +from antelab.runs.errors import RunStateConflictError +from antelab.runs.model import RunRequest +from antelab.runs.ownership import validate_artifact_ownership +from antelab.runs.repository import RunRepository + + +class _CancelledSignalError(RuntimeError): + pass + + +class _TransientRepositoryIOError(RuntimeError): + pass + + +class RunEngine(Protocol): + def execute( + self, request: RunRequest, progress: Callable[[int], bool] + ) -> RunArtifact: ... + + +class DeterministicFakeEngine: + def __init__( + self, + artifact: RunArtifact, + *, + ticks: Iterable[int], + on_tick: Callable[[int], None] | None = None, + ) -> None: + self._artifact = artifact + self._ticks = tuple(ticks) + self._on_tick = on_tick + + def execute( + self, request: RunRequest, progress: Callable[[int], bool] + ) -> RunArtifact: + del request + for tick in self._ticks: + if self._on_tick is not None: + self._on_tick(tick) + if not progress(tick): + raise _CancelledSignalError + return self._artifact + + +class InProcessSimulationEngine: + def execute( + self, request: RunRequest, progress: Callable[[int], bool] + ) -> RunArtifact: + simulation = Simulation.create(request.config) + metrics_by_tick: dict[int, dict[str, int | str]] = {} + while simulation.status == "running": + result = simulation.step() + if not progress(result.tick): + raise _CancelledSignalError + if result.tick % 10 == 0 or result.status != "running": + metrics_by_tick[result.tick] = _metric(simulation, result.checksum) + artifact = RunArtifact.from_simulation( + simulation, + tuple(metrics_by_tick[tick] for tick in sorted(metrics_by_tick)), + ) + artifact.validate() + return artifact + + +class RunWorker: + def __init__( + self, + repository: RunRepository, + artifacts: ArtifactRepository, + engine: RunEngine, + *, + now: Callable[[], datetime], + token_factory: Callable[[], str], + lease_duration: timedelta, + ) -> None: + self._repository = repository + self._artifacts = artifacts + self._engine = engine + self._now = now + self._token_factory = token_factory + self._lease_duration = lease_duration + + def run_once(self) -> bool: + lease = self._repository.claim_next( + now=self._now(), + lease_duration=self._lease_duration, + token_factory=self._token_factory, + ) + if lease is None: + return False + active_lease = lease + try: + request = self._repository.request(active_lease.run_id) + except OSError: + return True + + def fail_or_cancel( + *, code: str, message: str, retryable: bool + ) -> None: + nonlocal active_lease + current = self._repository.get(active_lease.run_id) + active_lease = replace(active_lease, revision=current.revision) + if current.state == "cancelling": + self._repository.acknowledge_cancel( + active_lease, now=self._now() + ) + return + try: + self._repository.fail( + active_lease, + code=code, + message=message, + retryable=retryable, + now=self._now(), + ) + except RunStateConflictError: + current = self._repository.get(active_lease.run_id) + if current.state != "cancelling": + raise + active_lease = replace(active_lease, revision=current.revision) + self._repository.acknowledge_cancel( + active_lease, now=self._now() + ) + + def progress(tick: int) -> bool: + nonlocal active_lease + try: + snapshot = self._repository.get(active_lease.run_id) + except OSError as error: + raise _TransientRepositoryIOError from error + if snapshot.state == "cancelling": + return False + if tick % 10 != 0 and tick != request.config.ticks: + return True + try: + active_lease = self._repository.renew_lease( + active_lease, + now=self._now(), + lease_duration=self._lease_duration, + ) + progressed = self._repository.record_progress( + active_lease, tick=tick, now=self._now() + ) + active_lease = replace( + active_lease, revision=progressed.revision + ) + except OSError as error: + raise _TransientRepositoryIOError from error + except RunStateConflictError: + if self._repository.get(active_lease.run_id).state == "cancelling": + return False + raise + return True + + try: + artifact = self._engine.execute(request, progress) + except _TransientRepositoryIOError: + return True + except _CancelledSignalError: + current = self._repository.get(active_lease.run_id) + active_lease = replace(active_lease, revision=current.revision) + self._repository.acknowledge_cancel(active_lease, now=self._now()) + return True + except Exception: + fail_or_cancel( + code="engine_failure", + message="experiment engine failed", + retryable=False, + ) + return True + + try: + validate_artifact_ownership( + request, artifact + ) + except Exception: + fail_or_cancel( + code="artifact_request_mismatch", + message="artifact does not belong to the run request", + retryable=False, + ) + return True + + try: + verifying = self._repository.begin_verifying( + active_lease, now=self._now() + ) + active_lease = replace(active_lease, revision=verifying.revision) + except RunStateConflictError: + if self._repository.get(active_lease.run_id).state == "cancelling": + current = self._repository.get(active_lease.run_id) + active_lease = replace(active_lease, revision=current.revision) + self._repository.acknowledge_cancel(active_lease, now=self._now()) + return True + raise + try: + reference = self._artifacts.stage(artifact) + except OSError: + return True + except Exception: + fail_or_cancel( + code="artifact_validation_failed", + message="artifact validation or staging failed", + retryable=False, + ) + return True + publishing = self._repository.begin_publishing( + active_lease, + artifact_id=reference.artifact_id, + now=self._now(), + ) + active_lease = replace(active_lease, revision=publishing.revision) + try: + published = self._artifacts.publish(reference) + except OSError: + return True + except Exception: + fail_or_cancel( + code="artifact_publication_failed", + message="artifact publication failed", + retryable=False, + ) + return True + self._repository.complete( + active_lease, artifact=published, now=self._now() + ) + return True diff --git a/docs/adr/0001-background-experiment-runs.md b/docs/adr/0001-background-experiment-runs.md new file mode 100644 index 0000000..5bdf3f6 --- /dev/null +++ b/docs/adr/0001-background-experiment-runs.md @@ -0,0 +1,11 @@ +# Use background Experiment Runs instead of a 30-second release gate + +Status: accepted + +AnteLab treats a scientifically trustworthy 5,000-tick run taking about 100 +seconds as valuable when it executes in the background. The proven 120-second +local closure budget and 512 MiB memory budget remain operational gates; the +30-second target becomes non-blocking stretch telemetry. V1 therefore invests +in durable, observable, cancellable Experiment Runs rather than further +Python-only kernel rewrites, while preserving deterministic artifacts and +requiring separate evidence before making any cross-platform latency SLA. diff --git a/docs/superpowers/specs/2026-07-13-background-experiment-runs-architecture.md b/docs/superpowers/specs/2026-07-13-background-experiment-runs-architecture.md new file mode 100644 index 0000000..c0b1253 --- /dev/null +++ b/docs/superpowers/specs/2026-07-13-background-experiment-runs-architecture.md @@ -0,0 +1,317 @@ +# Background Experiment Runs Architecture + +Status: proposed implementation architecture; product contract accepted. + +## 1. Product Contract + +A 5,000-tick run taking about 100 seconds is a normal product outcome when the +caller does not have to remain connected and the Published Artifact remains +scientifically trustworthy. + +Hard product gates: + +- deterministic scientific semantics and byte-identical artifacts; +- durable submission, progress, cancellation intent, and terminal state; +- no Published Artifact before complete scientific validation; +- recovery with explicit attempt history and no false resume claim; +- local reference-machine closure at <=120 seconds and <=512 MiB; +- responsive status/cancel control while the run computes. + +The 120-second gate is an operational reference-machine budget already supported +by existing evidence, not a new cross-platform public SLA. The 30-second target +is stretch telemetry: record it, but never fail, cancel, retry, block release, or +classify an Experiment Run as unhealthy solely for exceeding it. + +Final background acceptance after implementation still requires separately +authorized 5,000-tick evidence because worker integration changes invalidate +older ref-bound runtime evidence. This architecture does not authorize that run. + +## 2. Selected Deep Module and Interface + +Create a deep `ExperimentRuns` Module at the Seam used by CLI, future local UI, +and tests: + +```python +class ExperimentRuns: + def submit( + self, + request: RunRequest, + *, + idempotency_key: IdempotencyKey, + ) -> RunSnapshot: ... + + def get(self, run_id: RunId) -> RunSnapshot: ... + + def cancel(self, run_id: RunId) -> RunSnapshot: ... + + def artifact(self, run_id: RunId) -> RunArtifact: ... +``` + +The four entries are all necessary. Returning only an artifact locator from +`get` would force callers to understand storage, digests, parsing, and scientific +revalidation. A product event stream is deferred until a second real caller +needs push delivery; V1 polls immutable Run Snapshots. + +The Interface never exposes worker identity, leases, fencing tokens, SQLite, +paths, staging files, retry policy, thread/process choice, output paths, +priority, checkpoint knobs, or the 30-second stretch target. + +`RunRequest` contains a normalized `SimulationConfig` and binds engine version +and artifact schema version through its canonical digest. It does not contain a +caller-selected filesystem path. `submit` validates and durably admits the Run; +it never waits for simulation execution. + +## 3. Domain Model + +```python +RunState = Literal[ + "queued", + "running", + "cancelling", + "completed", + "failed", + "cancelled", +] + +@dataclass(frozen=True) +class RunProgress: + attempt: int + attempt_tick: int + max_tick_seen: int + ticks_requested: int + updated_at: datetime + +@dataclass(frozen=True) +class ArtifactRef: + artifact_id: str + sha256: str + byte_size: int + schema_version: int + +@dataclass(frozen=True) +class RunSnapshot: + run_id: RunId + revision: int + request_digest: str + state: RunState + phase: Literal["waiting", "computing", "verifying", "publishing", "terminal"] + progress: RunProgress + artifact: ArtifactRef | None + failure: PublicFailure | None +``` + +`phase` explains progress within `running` without exposing internal states that +callers would have to recover from. Artifact is non-null only for `completed`; +failure is non-null only for `failed`. Exact integer progress satisfies +`0 <= attempt_tick <= ticks_requested` and `max_tick_seen >= attempt_tick`. + +Scientific engine statuses `completed`, `extinct`, and `capacity_exceeded` are +all successful Experiment Run outcomes if their artifacts validate. User +cancellation is not a scientific status and never creates a cancelled artifact. + +## 4. State and Race Contract + +Allowed visible transitions: + +```text +queued -> running -> completed + | | \\-> failed + | \\----> queued # expired lease, deterministic replay + | \\----> cancelling -> cancelled + \\-----------------------------> cancelled +``` + +Terminal states are `completed`, `failed`, and `cancelled`. + +- `queued -> running` is an atomic claim that creates a Run Attempt and fencing + token. +- Same-attempt progress is monotonic. Deterministic Replay increments attempt + and may reset `attempt_tick` to zero; `max_tick_seen` never decreases. +- Every progress, cancel acknowledgement, publish, and terminal write carries + the current lease/fencing token and expected revision. +- An expired/stale worker cannot renew, publish, complete, or overwrite progress. +- Default worker concurrency is one so CPU contention cannot invalidate runtime + evidence. Store semantics may support multiple workers later. + +Cancellation is cooperative at tick boundaries: + +- queued cancel commits directly to `cancelled`; +- running cancel wins a compare-and-swap to `cancelling`; the worker finishes + the current tick, acknowledges, and publishes no artifact; +- repeated cancel is idempotent; +- terminal cancel returns the existing snapshot; +- completion and cancel race through one compare-and-swap. If cancellation wins, + artifact publication is forbidden. If the worker first enters the verifying + phase with a valid fence, cancel is too late and returns the current snapshot. + +No hard wall-time cancellation guarantee is claimed. Expected cancellation +latency is the current tick plus one repository round trip. A stuck tick requires +lease expiry and process supervision, not unsafe thread termination. + +## 5. Idempotency + +The idempotency namespace is local to one AnteLab runtime/database. + +1. Normalize and validate the request. +2. Digest canonical config, engine version, and artifact schema version. +3. A new `(namespace, idempotency_key)` atomically creates one queued Run. +4. Same key and same digest returns the existing Run Snapshot, including a + terminal one, and never creates another computation. +5. Same key and different digest raises `IdempotencyConflict`. +6. Concurrent equal submissions produce one Run ID. +7. V1 has no public retry operation. A user-requested new logical Run requires a + new key. Automatic recovery only handles lost infrastructure attempts. + +Idempotency keys never enter the scientific artifact or affect RNG/run identity. + +## 6. Recovery + +V1 has no stable checkpoint schema. Recovery is Deterministic Replay from tick +zero, never resume. + +- queued Run remains queued; +- running with an unexpired lease remains owned; +- running with an expired lease and no cancellation returns to queued, increments + recovery count, and later starts a new attempt at tick zero; +- cancelling with an expired lease becomes cancelled; +- completed/failed/cancelled never auto-retry; +- repeated worker loss is bounded; exhaustion becomes failed with stable + `worker_recovery_exhausted` fingerprint. + +The implementation budget will calibrate lease duration, heartbeat cadence, and +attempt cap with fake-clock failure tests. These deployment values do not enter +the product Interface. Progress is persisted at most every ten ticks or roughly +once per second, whichever comes first, and is advisory rather than a checkpoint. + +Computation is at-least-once after process loss. Lease fencing, deterministic +replay, idempotent publication, and state compare-and-swap guarantee one +authoritative caller-visible result; exactly-once computation is not claimed. + +## 7. Atomic Published Artifact + +Use a private `ArtifactRepository` Seam. The durable Adapter publishes only +canonical validated bytes inside a managed root; callers cannot select paths. + +1. Construct `RunArtifact` and run `validate()`. +2. Serialize canonical bytes once, parse/revalidate, and calculate SHA-256. +3. Write a same-filesystem unique staging file; flush and fsync it. +4. With the current fence and no winning cancellation, record the candidate and + enter `running.phase=publishing`. +5. Publish to a content-addressed immutable path without overwriting existing + content; fsync the directory. +6. Compare-and-swap the Run to completed with its ArtifactRef. +7. Only after that Run commit may `get` expose ArtifactRef or `artifact()` return + a value. + +SQLite and filesystem cannot share a transaction. Crash reconciliation closes +the window: + +- before final publish: validate staging or replay; +- final exists but Run is not completed: verify digest/request binding and finish + the state commit; +- completed: repeated completion returns the same ArtifactRef; +- conflicting bytes at an existing artifact identity: fail without overwrite; +- orphan staging/content is retained for separately authorized cleanup. + +`artifact(run_id)` requires completed state, verifies stored SHA-256, strictly +parses and validates `RunArtifact`, reserializes byte-identically, then returns +the frozen value. It never returns a partial or best-effort result. + +If later retrieval detects missing or corrupt storage, `artifact()` raises a +stable `ArtifactIntegrityError` and records an operational integrity incident; +it does not rewrite scientific history or silently change the completed Run into +a different lifecycle outcome. Repair, rebuild, or deletion requires a separate +human-gated operation. + +## 8. Internal Seams and Adapters + +Dependencies and Locality: + +```text +CLI / future local UI + -> ExperimentRuns Interface + -> RunRepository Seam + <- InMemoryRunRepository Adapter + <- SqliteRunRepository Adapter + -> ArtifactRepository Seam + <- InMemoryArtifactRepository Adapter + <- FilesystemArtifactRepository Adapter + +local supervisor -> RunWorker + -> RunRepository + -> ArtifactRepository + -> RunEngine Seam + <- DeterministicFakeEngine Adapter + <- InProcessSimulationEngine Adapter +``` + +The simulation remains authoritative and in-process. The fake engine is needed +to deterministically exercise cancellation, crashes, and races, making the +internal RunEngine Seam real. The production Adapter executes +`Simulation.step()` and existing artifact validation; it does not use either +retired performance prototype. + +Durable V1 composition uses Python stdlib SQLite plus a managed local filesystem. +Memory Adapters run the same contract suites. Do not add Redis, Celery, Kafka, +HTTP/WebSocket, cloud storage, multiple active workers, native kernels, or +checkpoint persistence in V1. + +This Module has high Depth: four caller operations hide admission, idempotency, +CAS, progress, cancellation, leases, replay, validation, publication, and +reconciliation. Removing it would scatter those rules across CLI, worker, and +future UI, so it passes the deletion test and provides Locality. + +## 9. Failure Contract + +Submission rejects invalid config/key without creating a Run. Stable public +failure classes distinguish: + +- permanent scientific/contract failures: engine invariant, checksum mismatch, + artifact validation, artifact identity conflict; +- recoverable infrastructure attempt failures: worker loss, lease expiry, + transient SQLite/artifact I/O; +- recovery exhaustion: terminal failed; +- cancellation: terminal cancelled, not failed. + +Unknown engine exceptions default to permanent failure so deterministic bugs do +not retry forever. Public failures contain a stable code, bounded sanitized +message, retryability, attempt, and last progress. Tracebacks, absolute paths, +tokens, and raw exception representations stay in controlled local logs. + +## 10. Revised Queue 2 Acceptance + +Queue 2 is no longer a kernel speed loop. Its previous verifier and 30-second +failure fingerprint are retired as historical evidence. + +Implementation-readiness gates, without a 5,000-tick run: + +- all state transitions and forbidden transitions are machine-tested; +- same-key idempotency, conflicting-key rejection, and concurrent submit pass; +- cancel-before-claim, running cancel, duplicate cancel, completion race, and + terminal cancel pass; +- stale fencing tokens cannot update progress or publish; +- restart/crash windows around claim, progress, staging, publish, and completion + reconcile to one explainable terminal or replay state; +- memory and durable Adapter contract suites agree; +- deterministic short-run artifact is byte-identical to synchronous reference; +- invalid/partial/corrupt artifacts are never exposed; +- progress remains queryable and truthful across replay; +- full tests, Ruff, strict targeted mypy, dependency lock, and independent + checker pass. + +Final product acceptance, separately human-gated: + +- one background 5,000-tick run on the declared reference machine; +- <=120 seconds and <=512 MiB; +- status and cancellation remain responsive while running; +- final artifact bytes/checksum equal the synchronous reference contract; +- 30-second result is recorded only as stretch telemetry. + +## 11. Strongest Counterargument + +For a single developer, shell backgrounding the current CLI is much simpler. +This architecture earns its complexity only because the accepted product needs +idempotent submission, truthful progress, cooperative cancellation, restart +recovery, and a scientifically validated artifact publication point. V1 keeps +the cost bounded by remaining single-machine, single-worker, SQLite/filesystem, +polling-only, and checkpoint-free. diff --git a/goals/antelab-background-experiment-runs.goal.md b/goals/antelab-background-experiment-runs.goal.md new file mode 100644 index 0000000..500c8bd --- /dev/null +++ b/goals/antelab-background-experiment-runs.goal.md @@ -0,0 +1,89 @@ +# Goal: AnteLab Background Experiment Runs + +Status: `stopped-limit` + +Work in: `/private/tmp/antelab-v1-performance-gate-worktree` + +Canonical design/state: + +- `docs/superpowers/specs/2026-07-13-background-experiment-runs-architecture.md` +- `CONTEXT.md` +- `docs/adr/0001-background-experiment-runs.md` +- `state/antelab-background-experiment-runs.STATE.md` + +## Outcome + +Implement a single-machine, single-worker `ExperimentRuns` Module whose +`submit/get/cancel/artifact` Interface provides durable idempotency, truthful +progress, tick-boundary cancellation, deterministic replay after worker loss, +and unique immutable Published Artifact retrieval. + +Implementation readiness succeeds without a 5,000-tick run when state/race/ +crash contracts, memory/durable Adapter parity, short-run artifact equality, +full tests/static checks, and independent checker all pass. + +## Control Shape + +- Verdict: `goal` with unlike domain, concurrency, persistence, worker, and + verification gates; not a performance loop. +- Execution authority after approval: `run-to-stop` for implementation + readiness only. +- Controller concurrency: one writer and one active implementation round. + +## Scope + +- In scope: domain types/state machine; `ExperimentRuns`; RunWorker; + deterministic fake and in-process engine Adapters; memory/SQLite RunRepository + Adapters; memory/filesystem ArtifactRepository Adapters; focused tests, + verifier, state, and reports. +- Integration may use short synthetic configs and the existing 500-tick golden. +- Protected: scientific transition semantics, artifact schema/golden, retired + prototypes/evidence, dependency lock except stdlib use, synchronous CLI + behavior/default, public 120-second/512 MiB operational gate, and 30-second + stretch classification. +- Non-goals: 5,000 ticks, HTTP/WebSocket/UI, accounts, multi-host/distributed + queue, multiple active workers, checkpoint resume, automatic destructive GC, + native acceleration, default switch, commit, push, merge, or deployment. + +## Gates + +1. Domain gate: exhaustive allowed/forbidden transitions, revisions, progress, + idempotency, and cancellation semantics through memory Adapters. +2. Race gate: deterministic fake engine proves one CAS winner, stale-fence + rejection, replay attempt semantics, and bounded recovery. +3. Durability gate: SQLite/filesystem contract parity plus injected crashes at + claim, progress, staging, immutable publish, and completion reconciliation. +4. Engine gate: short in-process reference execution, responsive polling/cancel, + byte-identical artifact, corrupt/partial rejection, and retrieval revalidation. +5. Acceptance gate: full tests, Ruff, targeted strict mypy, dependency/scope + locks, known-good/known-bad verifier calibration, and independent checker. + +## Progress and Evidence + +- Progress metric: passed contract scenarios / locked scenario manifest. +- Failure fingerprint: normalized failed state/race/crash/artifact assertions. +- Locked verifier: created in bootstrap before product mutation; mutator may not + weaken it. Verifier repair requires a separate round and checker calibration. +- Evidence: append-only reports under + `reports/goals/antelab-background-experiment-runs/`, bound to exact worktree, + HEAD/WIP fingerprint, commands, environment, and timestamps. + +## Limits and Stops + +- Proposed cap reuses the existing controller ceiling: 4 implementation rounds, + 6 hours total, and one failure cluster per round. +- Test workload: at most 500 ticks; zero 5,000-tick executions. +- Stop after two rounds with the same fingerprint and zero positive scenario + delta; stop immediately on duplicate authoritative completion, stale-fence + commit, partial artifact exposure, unexplained state/event divergence, path + escape, or a recovery requirement that needs checkpoint semantics. +- Human gates: verifier contract changes, dependencies beyond stdlib, cleanup, + 5,000 ticks, commit/push/merge, CLI/default switch, publication, and deploy. + +## Handoff + +- State: `state/antelab-background-experiment-runs.STATE.md`. +- Each round reads state first, handles one failure cluster, runs the locked + evaluator, records exact evidence, and selects exactly one next focus. +- On implementation readiness success, stop `waiting-human` for separate CLI + integration and one background 5,000-tick acceptance authorization. diff --git a/goals/background-runs-precommit-closure.goal.md b/goals/background-runs-precommit-closure.goal.md new file mode 100644 index 0000000..6366b76 --- /dev/null +++ b/goals/background-runs-precommit-closure.goal.md @@ -0,0 +1,35 @@ +# Goal: Background Runs Pre-commit Closure + +Status: `complete` + +- Authority: run-to-stop, maximum 2 rounds. +- Objective: close the six Spec/Standards blockers found by the scoped + pre-commit review without broad refactoring. +- Round 1: behavioral contracts and minimal product/state fixes. +- Round 2: aggregate verification, independent checker, and durable closure. +- Allowed mutation: `antelab/runs/**`, `tests/runs/**`, background-run verifier, + manifests, Goal/STATE/reports, and the canonical background-run STATE. +- Protected: synchronous CLI/default runner, scientific core, dependencies, + compact/headless/roadmap WIP, commit/push/deploy/cleanup. +- Test workload: every executed simulation at most 500 ticks; zero 5,000-tick + executions. +- Progress: unresolved pre-commit blockers decrease from 6 to 0. +- Stop: success, repeated no-improvement fingerprint, round limit, scope + expansion, or a new human/Git gate. +- State: `state/background-runs-precommit-closure.STATE.md`. + +Acceptance: + +1. Concurrent equal submissions prove one Run ID on memory and SQLite. +2. Real in-process execution proves observable polling and cooperative cancel. +3. Progress persistence is bounded to every ten ticks or terminal tick. +4. Transient repository I/O is never classified as permanent engine failure. +5. Canonical background-run STATE reflects the repair supersession. +6. Tick-guarded aggregate Ruff, mypy, and pytest pass, followed by independent + Standards/Spec checker PASS. + +Completion: all six blockers closed; Ruff, strict mypy, and the isolated scoped +suite's 590 tests pass with the global 500-step guard. The earlier 619-test run +also included protected unrelated WIP; independent Standards and Spec checkers +both PASS. +Stopped at the explicit commit/default-runner human gate. diff --git a/goals/background-runs-readiness-repair.goal.md b/goals/background-runs-readiness-repair.goal.md new file mode 100644 index 0000000..51ba1d1 --- /dev/null +++ b/goals/background-runs-readiness-repair.goal.md @@ -0,0 +1,17 @@ +# Goal: Background Runs Readiness Repair + +Status: `complete` + +- Authority: run-to-stop, maximum 3 rounds / 4 hours. +- Round 1: verifier-repair only; product implementation is protected. +- Round 2-3 only after independent verifier checker PASS. +- Test workload: at most 500 ticks; zero 5,000-tick executions. +- Protected: synchronous CLI/default runner, artifact/scientific contracts, + dependencies, commit/push/deploy/cleanup. +- State: `state/background-runs-readiness-repair.STATE.md`. +- Success: behavioral verifier known-good/known-bad and independent checker PASS, + then all four remaining implementation clusters close and final checker PASS. + +Completion evidence: Round 1 verifier checker PASS; Round 2 CAS/race checker +PASS; Round 3 staging/I/O checker PASS; final readiness verifier 31/31 and +calibration 4/4; Ruff and mypy PASS. Stopped before Git/default-runner gates. diff --git a/reports/goals/antelab-background-experiment-runs/round-01.md b/reports/goals/antelab-background-experiment-runs/round-01.md new file mode 100644 index 0000000..927bc56 --- /dev/null +++ b/reports/goals/antelab-background-experiment-runs/round-01.md @@ -0,0 +1,30 @@ +# Background Experiment Runs — Round 01 + +## Focus + +Lock the scenario manifest and implement the domain/state machine through the +approved `ExperimentRuns` and in-memory RunRepository Seams. + +## TDD Evidence + +- Missing `antelab.runs` produced the expected initial import failure. +- First green exposed a hidden side effect: duplicate submit consumed a Run ID. +- ID generation moved inside the atomic create branch; repeat submit is now + side-effect free and conflicting reuse raises `IdempotencyConflictError`. +- Five focused tests pass and cover eight of nineteen manifest scenarios: + idempotency pair, queued cancel, exclusive claim, stale progress fence, + cancel/completion CAS outcomes, and deterministic replay progress semantics. + +## Verification + +- Focused pytest: 5 passed. +- Ruff behavior rules are clean after adopting repository-standard `Error` + suffixes and import ordering. +- No simulation or artifact publication ran. +- Maximum configured test request: 100 ticks; executed ticks: zero. +- 5,000-tick executions: zero. + +## Decision + +Gate 1 passes. Continue to the fake-engine worker/race gate. No commit, push, +CLI/default change, dependency change, deployment, or cleanup occurred. diff --git a/reports/goals/antelab-background-experiment-runs/round-02.md b/reports/goals/antelab-background-experiment-runs/round-02.md new file mode 100644 index 0000000..c1ffbe7 --- /dev/null +++ b/reports/goals/antelab-background-experiment-runs/round-02.md @@ -0,0 +1,29 @@ +# Background Experiment Runs — Round 02 + +## Focus + +Implement the deterministic fake-engine RunWorker path and observable +publication/cancellation behavior without durable I/O. + +## Evidence + +- RunWorker claims exactly one queued Run and records fenced progress. +- A three-tick fake execution passes through computing, verifying, publishing, + and completed; `artifact()` returns the validated value only afterward. +- A cancellation injected at tick two wins the state CAS, is acknowledged at + the tick boundary, and produces no ArtifactRef. +- In-memory staging validates canonical bytes and SHA-256 before immutable + publication; completed state binds the exact candidate identity. +- Seven cumulative focused tests pass; twelve of nineteen manifest scenarios + now have direct evidence. + +## Verification + +- Maximum executed simulation workload: three ticks for test fixture creation. +- 5,000-tick executions: zero. +- No synchronous CLI/default, scientific transition, artifact schema, dependency, + commit, push, deployment, or cleanup change occurred. + +## Decision + +Gate 2 passes. Continue to durable Adapter parity and restart/crash windows. diff --git a/reports/goals/antelab-background-experiment-runs/round-03.md b/reports/goals/antelab-background-experiment-runs/round-03.md new file mode 100644 index 0000000..5812180 --- /dev/null +++ b/reports/goals/antelab-background-experiment-runs/round-03.md @@ -0,0 +1,30 @@ +# Background Experiment Runs — Round 03 + +## Focus + +Implement durable local Adapters and close restart windows around immutable +artifact publication. + +## Evidence + +- SQLite RunRepository survives Adapter/process reconstruction with request, + snapshot, cancellation, lease, progress, and candidate identity intact. +- Filesystem ArtifactRepository keeps staging invisible, publishes under + content-addressed identity without overwrite, and reloads canonical validated + `RunArtifact` values after reconstruction. +- Crash after final publish but before Run completion reconciles exactly once to + completed; a second reconciliation is a no-op. +- Corrupt published bytes are rejected with `ArtifactIntegrityError`. +- Four durable focused tests pass; seventeen of nineteen manifest scenarios now + have direct evidence. + +## Verification + +- Durable tests use three-tick artifacts only. +- No 5,000-tick execution, dependency addition, CLI/default change, commit, + push, deployment, or cleanup occurred. + +## Decision + +Gate 3 passes. Continue to authoritative in-process engine integration and final +readiness verification. diff --git a/reports/goals/antelab-background-experiment-runs/round-04.md b/reports/goals/antelab-background-experiment-runs/round-04.md new file mode 100644 index 0000000..570c636 --- /dev/null +++ b/reports/goals/antelab-background-experiment-runs/round-04.md @@ -0,0 +1,76 @@ +# Background Experiment Runs — Round 04 + +## Focus + +Integrate the authoritative short-run engine, calibrate the structural verifier, +run full verification, and obtain independent Standards/Spec review. + +## Product Evidence + +- InProcessSimulationEngine executes the unchanged `Simulation.step()` oracle. +- A 25-tick background Published Artifact is byte-identical to synchronous + `run_experiment` output. +- Worker heartbeats renew leases; expired fences are rejected. +- Unknown engine and artifact ownership failures become sanitized permanent + failures rather than accidental replay. +- Both artifact Adapters parse/revalidate canonical bytes; filesystem stage and + directory publication are fsynced. +- SQLite metadata has a locked schema-version envelope and runtime model + invariants. +- Published-artifact crash reconciliation verifies request ownership and has a + dedicated expired-lease finalization path. + +## Verification + +- Full pytest: 597 passed. +- Focused readiness/verifier pytest: 18 passed. +- Ruff: passed. +- Targeted strict mypy: passed. +- Structural verifier known-good: PASS. +- Structural verifier known-bad missing-scenario calibration: PASS. +- `git diff --check`: passed. +- Maximum executed test workload: 25 ticks per execution; no test exceeds 500. +- 5,000-tick executions: zero. + +## Standards Review + +Independent review initially reported missing lease heartbeat, failure +transitions, request ownership, fsync, versioned durable serialization, and +runtime invariants, plus inheritance/duplication smells. The implementation +subsequently added heartbeat/expiry checks, sanitized failure transitions, +shared ownership validation, file/directory fsync, schema versioning, model +invariants, canonical-byte memory storage, and repository/artifact Protocols. + +The Standards re-review still returned `FAIL` on two P1 findings: ownership was +then missing on the normal worker path, and expired publication recovery could +not finalize. Both received local fixes after that review, but the four-round +budget ended before another independent Standards pass. The SQLite Adapter's +inheritance/forwarding smell remains non-blocking technical debt. + +## Spec Review + +Independent Spec re-review remained `FAIL`. It confirmed many earlier fixes but +left these readiness blockers: + +- expected snapshot revision is not part of worker CAS enforcement; +- publish failure before a final immutable object has no staging replay path; +- cancellation followed by engine failure can become failed instead of cancelled; +- transient artifact I/O is classified as permanent failure; +- memory/SQLite and memory/filesystem parity are not one shared contract suite; +- the structural verifier uses source tokens and can pass without proving those + behavioral contracts. + +Normal-path ownership and expired publication finalization were fixed after the +review, but the remaining findings were not authorized a fifth round. + +## Decision + +`stopped-limit`, not implementation-ready. The structural verifier PASS is a +false positive relative to the independent checker and cannot authorize CLI +integration or a 5,000-tick acceptance run. A separate verifier-repair/product +round is required if work continues. + +## Scope Audit + +No synchronous CLI/default change, 5,000-tick run, dependency addition, commit, +push, merge, deployment, publication, or cleanup occurred. diff --git a/reports/goals/background-runs-precommit-closure-round-01.md b/reports/goals/background-runs-precommit-closure-round-01.md new file mode 100644 index 0000000..99dcf39 --- /dev/null +++ b/reports/goals/background-runs-precommit-closure-round-01.md @@ -0,0 +1,18 @@ +# Background Runs Pre-commit Closure — Round 01 + +Verdict: `PASS` + +- Concurrent equal submissions create one Run ID for memory and SQLite. +- A real 500-tick-budget in-process worker remains observable and cooperatively + cancellable while executing. +- Progress/lease persistence occurs at ticks 10, 20, and terminal 25 rather + than on every tick. +- Transient repository I/O remains recoverable and is not stored as permanent + `engine_failure`. +- Canonical stopped-limit STATE now explicitly points to superseding evidence. +- Readiness verifier: 36 passed. +- Aggregate gate: Ruff PASS; strict mypy 38 files PASS; pytest 619 / 619 PASS. +- Test execution used a global guard that fails step 501 per simulation; + maximum observed budget 500 and zero 5,000-tick executions. +- No CLI/default-runner, dependency, scientific-core, commit, push, deployment, + or cleanup action occurred. diff --git a/reports/goals/background-runs-precommit-closure-round-02.md b/reports/goals/background-runs-precommit-closure-round-02.md new file mode 100644 index 0000000..e7a7f4a --- /dev/null +++ b/reports/goals/background-runs-precommit-closure-round-02.md @@ -0,0 +1,21 @@ +# Background Runs Pre-commit Closure — Round 02 + +Verdict: `PASS` + +- Standards checker: PASS; command-for-command `make verify` equivalence is + proven under the stricter global 500-step guard. +- Spec checker: PASS; all six pre-commit blockers closed. +- Readiness verifier: 36 / 36 passed. +- Focused concurrency/worker/CAS/durability: 34 / 34 passed independently. +- Pre-isolation aggregate gate: Ruff PASS; strict mypy 38 files PASS; pytest + 619 / 619 PASS. +- Recovered isolated commit gate: readiness 36 / 36; Ruff PASS; strict mypy 32 + files PASS; pytest 590 / 590 PASS. The 29-test delta is excluded protected + compact/headless/performance WIP, not lost background coverage. +- Four prior readiness-repair clusters remain green. +- CLI/default runner, dependencies, artifact/scientific core, and unrelated WIP + remain protected. +- Maximum simulation budget: 500 ticks; zero 5,000-tick executions. +- No commit, push, deployment, cleanup, or default switch. + +Stop: complete at the explicit commit/default-runner human gate. diff --git a/reports/goals/background-runs-readiness-repair-round-01.md b/reports/goals/background-runs-readiness-repair-round-01.md new file mode 100644 index 0000000..2d95863 --- /dev/null +++ b/reports/goals/background-runs-readiness-repair-round-01.md @@ -0,0 +1,18 @@ +# Background Runs Readiness Repair — Round 01 + +Verdict: `PASS` + +- Scope: verifier repair and executable behavioral contracts only. +- Behavioral verifier: 22 passed. +- Verifier calibration: 4 passed. +- Adapter contracts: memory/SQLite and memory/filesystem parity. +- Rejected known-bads: stale fencing token, visible staging artifact, modified + protected file, added product file, and missing scenario. +- Scope proof: SHA-256 baseline covers all 32 `antelab/**/*.py` files plus + `pyproject.toml` and `uv.lock`, with exact product path-set equality. +- Tick proof: runtime guard rejects step 501 per simulation; observed maximum 25. +- Independent checker: PASS on third review. +- Prohibited actions: no 5,000-tick run, commit, push, deploy, or default CLI change. + +Round 2 unlock: only expected-revision CAS, staging recovery, cancel/error race, +and transient I/O classification may mutate product code. diff --git a/reports/goals/background-runs-readiness-repair-round-02.md b/reports/goals/background-runs-readiness-repair-round-02.md new file mode 100644 index 0000000..f570c6f --- /dev/null +++ b/reports/goals/background-runs-readiness-repair-round-02.md @@ -0,0 +1,14 @@ +# Background Runs Readiness Repair — Round 02 + +Verdict: `PASS` + +- Expected-revision CAS is enforced together with attempt and fencing token. +- Worker advances revision after progress, verifying, and publishing writes. +- SQLite persists snapshot and lease revisions transactionally. +- Repository forbids `cancelling -> failed`. +- A shared CAS-safe worker path resolves engine, ownership, and staging errors + without overwriting a cancellation winner. +- Focused regression: 23 passed; maximum simulation workload 4 ticks. +- Independent checker: PASS on second review. +- No 5,000-tick run, CLI/default-runner change, dependency change, commit, push, + or deployment. diff --git a/reports/goals/background-runs-readiness-repair-round-03.md b/reports/goals/background-runs-readiness-repair-round-03.md new file mode 100644 index 0000000..a8ca9cf --- /dev/null +++ b/reports/goals/background-runs-readiness-repair-round-03.md @@ -0,0 +1,20 @@ +# Background Runs Readiness Repair — Round 03 + +Verdict: `PASS` + +- Filesystem staging survives Adapter restart and is integrity checked, + atomically published, ownership validated, and finalized exactly once. +- Stage `OSError` remains `running/verifying` for expired-lease replay. +- Publish `OSError` remains `running/publishing` for reconciliation. +- Permanent worker publication errors become sanitized + `artifact_publication_failed` terminal failures. +- Corrupt staging and wrong-request recovery become sanitized + `artifact_recovery_failed` terminal failures. +- Recovery completion/failure ignores lease expiry intentionally, but retains + attempt, fencing-token, and expected-revision CAS. +- Final readiness verifier: 31 passed; verifier calibration: 4 passed. +- Ruff and mypy: PASS across 18 source files. +- Independent final checker: PASS; no remaining P0/P1 blocker. +- Maximum observed simulation workload: 25 ticks; zero 5,000-tick executions. +- No synchronous CLI/default-runner change, dependency change, commit, push, + deployment, or cleanup. diff --git a/scripts/validate_background_runs_readiness.py b/scripts/validate_background_runs_readiness.py new file mode 100644 index 0000000..7eee269 --- /dev/null +++ b/scripts/validate_background_runs_readiness.py @@ -0,0 +1,136 @@ +"""Executable verifier for background Experiment Run readiness.""" + +from __future__ import annotations + +import argparse +import hashlib +import subprocess +import sys +from pathlib import Path + +EXPECTED_SCENARIOS = { + line.strip() + for line in """ +submit_same_key_same_request_deduplicates +submit_same_key_different_request_conflicts +concurrent_equal_submissions_create_one_run +cancel_queued_is_terminal_and_idempotent +claim_has_single_fenced_owner +stale_fence_cannot_progress_or_complete +running_cancel_beats_completion +completion_beats_late_cancel +expired_attempt_replays_from_tick_zero +recovery_exhaustion_fails +memory_sqlite_repository_contract_parity +memory_filesystem_artifact_contract_parity +staged_artifact_is_not_visible +immutable_publish_deduplicates_same_bytes +immutable_publish_rejects_conflicting_bytes +publish_then_crash_reconciles_once +short_run_progress_is_truthful +short_run_cancel_publishes_no_artifact +short_run_artifact_matches_reference_bytes +real_inprocess_run_is_observable_and_cancellable +progress_persistence_is_bounded +transient_repository_io_is_recoverable +corrupt_artifact_is_never_returned +""".splitlines() + if line.strip() +} + +REQUIRED_FILES = { + "antelab/runs/model.py", + "antelab/runs/manager.py", + "antelab/runs/repository.py", + "antelab/runs/sqlite_repository.py", + "antelab/runs/artifacts.py", + "antelab/runs/worker.py", + "antelab/runs/recovery.py", +} + +ROUND_ONE_BASELINE = "tests/manifests/background-runs-round-one.sha256" + + +BEHAVIORAL_TESTS = ( + "tests/runs/test_adapter_contracts.py", + "tests/runs/test_experiment_runs.py", + "tests/runs/test_run_repository.py", + "tests/runs/test_run_worker.py", + "tests/runs/test_durable_adapters.py", + "tests/runs/test_inprocess_engine.py", +) + + +def validate_structure(root: Path) -> tuple[str, ...]: + errors: list[str] = [] + manifest_path = root / "tests/manifests/background-run-scenarios.txt" + manifest = { + line.strip() + for line in manifest_path.read_text(encoding="utf-8").splitlines() + if line.strip() + } + if manifest != EXPECTED_SCENARIOS: + errors.append("scenario manifest differs from the locked expected set") + missing_files = sorted(path for path in REQUIRED_FILES if not (root / path).is_file()) + if missing_files: + errors.append(f"required files missing: {missing_files}") + return tuple(errors) + + +def validate_round_one_scope(root: Path) -> tuple[str, ...]: + errors: list[str] = [] + baseline_path = root / ROUND_ONE_BASELINE + if not baseline_path.is_file(): + errors.append("Round 1 protected-file baseline is missing") + return tuple(errors) + baseline_product_paths: set[str] = set() + for line in baseline_path.read_text(encoding="utf-8").splitlines(): + expected, relative = line.split(maxsplit=1) + if relative.startswith("antelab/"): + baseline_product_paths.add(relative) + protected = root / relative + if not protected.is_file(): + errors.append(f"Round 1 protected file missing: {relative}") + continue + actual = hashlib.sha256(protected.read_bytes()).hexdigest() + if actual != expected: + errors.append(f"Round 1 protected file changed: {relative}") + actual_product_paths = { + path.relative_to(root).as_posix() + for path in (root / "antelab").rglob("*.py") + } + if actual_product_paths != baseline_product_paths: + errors.append("Round 1 product file set changed") + return tuple(errors) + + +def run_behavioral_contract(root: Path) -> int: + return subprocess.run( + [sys.executable, "-m", "pytest", "-q", *BEHAVIORAL_TESTS], + cwd=root, + check=False, + ).returncode + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--round-one-scope", action="store_true") + args = parser.parse_args(argv) + root = args.root.resolve() + errors = validate_structure(root) + if args.round_one_scope: + errors += validate_round_one_scope(root) + if errors: + for error in errors: + print(f"FAIL: {error}") + return 1 + if run_behavioral_contract(root) != 0: + print("FAIL: executable background Run behavioral contract failed") + return 1 + print("PASS: executable background Experiment Run contract is locked") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/state/antelab-background-experiment-runs.STATE.md b/state/antelab-background-experiment-runs.STATE.md new file mode 100644 index 0000000..2609ccd --- /dev/null +++ b/state/antelab-background-experiment-runs.STATE.md @@ -0,0 +1,42 @@ +# State: AnteLab Background Experiment Runs + +Schema version: 1 + +- Status: `stopped-limit`. +- Execution authority: four-round implementation budget consumed; no further + mutation authorized. +- Worktree: `/private/tmp/antelab-v1-performance-gate-worktree`. +- Branch: `feature/v1-performance-gate`. +- Design: `docs/superpowers/specs/2026-07-13-background-experiment-runs-architecture.md`. +- Goal: `goals/antelab-background-experiment-runs.goal.md`. +- Product contract: ~100-second background runs are valuable; <=120 seconds and + <=512 MiB remain reference-machine operational gates; 30 seconds is stretch. +- Rounds used: 4 / 4. +- Runtime used: less than 1 / 6 hours. +- Test tick maximum used: 100 / 500. +- 5,000-tick executions used: 0 / 0. +- Scenario evidence: structural verifier reports 19 / 19 tokens, but the + independent checker rejected several mappings as insufficient contract proof; + readiness scenario count is therefore not accepted. +- Verification: 597 full tests passed; 18 focused readiness/verifier tests + passed; Ruff, targeted strict mypy, and `git diff --check` passed. +- Independent checker: `FAIL` on both Standards and Spec axes during Round 4. +- Current failure fingerprint: + `independent_checker_high_findings+verifier_false_positive`. +- Remaining hard findings: expected-revision CAS is not enforced; publish-before- + final recovery cannot replay staging; cancel-vs-engine-error can misclassify + cancellation; transient artifact I/O is classified permanent; memory/durable + contract parity is not exercised by one shared suite. +- Protected evidence: no 5,000-tick run, CLI/default change, dependency addition, + commit, push, deployment, or cleanup occurred. +- Next focus: human decision on a separate verifier-repair/implementation goal. + This stopped-limit assessment is historical and was superseded by + `background-runs-readiness-repair` and `background-runs-precommit-closure`. +- Superseding evidence: expected-revision CAS, staging recovery, cancel/error + race, transient artifact/repository I/O, shared Adapter contracts, concurrent + idempotency, bounded progress persistence, and real-engine cancellation are + closed; the recovered isolated commit gate passes Ruff, strict mypy over 32 + source files, and 590 / 590 tests. The earlier 619-test evidence included + protected unrelated WIP and is retained only as historical evidence. +- Current integration boundary: implementation-ready pending the independent + pre-commit checker and explicit Git/default-runner human gates. diff --git a/state/background-runs-precommit-closure.STATE.md b/state/background-runs-precommit-closure.STATE.md new file mode 100644 index 0000000..9d7d0fa --- /dev/null +++ b/state/background-runs-precommit-closure.STATE.md @@ -0,0 +1,24 @@ +# State: Background Runs Pre-commit Closure + +Schema version: 1 + +- Status: `complete`. +- Round: 2 / 2. +- Initial blockers: 6. +- Remaining blockers: 0. +- Maximum observed simulation ticks: 500 / 500. +- 5,000-tick executions: 0 / 0. +- Failure fingerprint: `precommit_spec_gap`. +- Authority: behavioral contracts and minimal product/state fixes. +- Round 1 evidence: concurrent memory/SQLite idempotency PASS; real in-process + polling/cancel PASS; progress cadence PASS; transient repository I/O PASS; + canonical STATE supersession recorded. +- Aggregate gate before WIP isolation: Ruff PASS; strict mypy 38 source files + PASS; pytest 619 / 619 PASS with global 500-step guard. +- Recovered isolated commit gate: readiness 36 / 36, Ruff PASS, strict mypy 32 + source files PASS, and pytest 590 / 590 with the same global 500-step guard. +- Independent Standards checker: `PASS`. +- Independent Spec checker: `PASS`; readiness 36/36, pre-isolation aggregate + 619/619, and + focused concurrency/worker/CAS/durability 34/34. +- Stop reason: objective achieved; waiting at commit/default-runner human gate. diff --git a/state/background-runs-readiness-repair.STATE.md b/state/background-runs-readiness-repair.STATE.md new file mode 100644 index 0000000..ee6b4ee --- /dev/null +++ b/state/background-runs-readiness-repair.STATE.md @@ -0,0 +1,30 @@ +# State: Background Runs Readiness Repair + +Schema version: 1 + +- Status: `complete`. +- Round: 3 / 3. +- Runtime: less than 1 / 4 hours. +- Authority: approved implementation repair clusters only. +- Product mutation: unlocked by independent verifier checker PASS. +- Maximum observed simulation ticks: 25 / 500; executable ceiling enabled. +- 5,000-tick executions: 0 / 0. +- Failure fingerprint: `structural_verifier_false_positive`. +- Independent checker attempt 1: `FAIL`; scope and tick controls were not executable. +- Independent checker attempt 2: `FAIL`; product path-set and artifact/scientific + protection were incomplete. +- Independent checker attempt 3: `PASS`; 34 baseline entries cover all 32 + `antelab/**/*.py` paths plus `pyproject.toml` and `uv.lock`; verifier 22/22, + calibration 4/4, maximum observed simulation 25 ticks. +- Round 2 checker attempt 1: `FAIL`; repository allowed `cancelling -> failed` + and ownership errors lacked cancel reconciliation. +- Round 2 checker attempt 2: `PASS`; expected-revision CAS and cancel/error race + contracts pass 23 focused tests. +- Next focus: durable staging recovery and transient artifact I/O classification. +- Round 3 checker attempt 1: `FAIL`; permanent publication integrity failures + could remain indefinitely in `running/publishing`. +- Round 3 checker attempt 2: `PASS`; transient stage/publish I/O remains + recoverable, while permanent publish/recovery failures terminate safely. +- Final verifier: 31 / 31 behavioral tests; calibration 4 / 4. +- Static verification: Ruff PASS; mypy PASS across 18 source files. +- Stop reason: goal achieved; waiting at explicit Git/default-runner human gate. diff --git a/tests/manifests/background-run-scenarios.txt b/tests/manifests/background-run-scenarios.txt new file mode 100644 index 0000000..62ba551 --- /dev/null +++ b/tests/manifests/background-run-scenarios.txt @@ -0,0 +1,23 @@ +submit_same_key_same_request_deduplicates +submit_same_key_different_request_conflicts +concurrent_equal_submissions_create_one_run +cancel_queued_is_terminal_and_idempotent +claim_has_single_fenced_owner +stale_fence_cannot_progress_or_complete +running_cancel_beats_completion +completion_beats_late_cancel +expired_attempt_replays_from_tick_zero +recovery_exhaustion_fails +memory_sqlite_repository_contract_parity +memory_filesystem_artifact_contract_parity +staged_artifact_is_not_visible +immutable_publish_deduplicates_same_bytes +immutable_publish_rejects_conflicting_bytes +publish_then_crash_reconciles_once +short_run_progress_is_truthful +short_run_cancel_publishes_no_artifact +short_run_artifact_matches_reference_bytes +real_inprocess_run_is_observable_and_cancellable +progress_persistence_is_bounded +transient_repository_io_is_recoverable +corrupt_artifact_is_never_returned diff --git a/tests/manifests/background-runs-round-one.sha256 b/tests/manifests/background-runs-round-one.sha256 new file mode 100644 index 0000000..3d1e64b --- /dev/null +++ b/tests/manifests/background-runs-round-one.sha256 @@ -0,0 +1,34 @@ +4059ae61c26515a4a9c8f69f8d044141670c8e77a851a6b019a549a43c37fa59 antelab/__init__.py +5d1722fa9b82a6bace7ea2d44570bd25db688b0fab539a5e0bb35fe1468b35f9 antelab/artifacts/__init__.py +dc0ca5b31eb30755289f0a3494b4a7022bf44d70dfb054c31f2c62683092b1b1 antelab/artifacts/schema.py +4b57da98ba49841d2a66f4667254ee2a685416f4c86efc56b908dfcd429cded1 antelab/artifacts/writer.py +d834f20c560e69ebd0029bf9f9966bc4fc5f6a3d68982ca7897a2543fc45164f antelab/cli.py +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 antelab/core/__init__.py +0e95c6fd40ae468766907894c20eb8ed5cf2a5792991db1f7b3fb1cc08fd071f antelab/core/brain.py +1c6dd16846de7e77d712fcb35c7d8e3d459131beb6ecdba01a00d663bd308c52 antelab/core/compact_kernel.py +63e1685770799227bd15e573d48fc9061cabd07a5b760dabc44daea5186c62f8 antelab/core/config.py +e52c94fe52291c232cb9bb7da5bac3fc135c3e417b2140baef775296afaca663 antelab/core/environment.py +f4eeda895ddb746427d5b4ea512fe47adca3b55840500cdebcc8f1d001b5dedc antelab/core/evolution.py +80cc482df0329c98a992fc0368b4e444190b069e39c36e5a89b69d1ed3dc717d antelab/core/fixed.py +d45bfac76cb7abf603363f904d83f99371560a7441cd03e5e176a665390a8f1b antelab/core/genome.py +490e3ec3042bcded932a832c6a4e3be1166f7ffab7fe4042194b2cee6405cc1f antelab/core/organism.py +d1367496fb792de2e1edacd8797bf164b66d40fc52b5137c4a69bf5c5858424e antelab/core/physics.py +6543b69bdd97b276d50d6e0dda0ac9df2ddcaeab24a433943520089a5756fea4 antelab/core/rng.py +415caff7993e87070402c52718e58ba3afcd4bc4a6b14557959be75e9ca854c9 antelab/core/simulation.py +79b0b72b8395f323bc1a75f9a32995066d86d4f3efddd618de1365b523ce670b antelab/core/spatial.py +adadeaff04a392ce1e9e750189f645a0a1be52150bb981d91e76a626066801d7 antelab/experiments/__init__.py +b8b51a050008578748d4a03425c1b39f6582a824b8a024cb875185d415594927 antelab/experiments/headless.py +446dbf15b984edcac6c4b0354c7a2fb62bdf9fdde07bbdc4d6b7612b97748cc9 antelab/experiments/headless_execution.py +63c2cec5988ac21eab6de035fa349612fac77b367fd4b247ff663f085af514c0 antelab/experiments/runner.py +afaf36c47c1c18edb93dcc1196127b00e7d54b664e95d0901f8e715b8e52ae4c antelab/runs/__init__.py +c62af1e381f8edc94e9c1e5632e8b12a2c73287fc88e4350084f7d76bebf80d5 antelab/runs/artifacts.py +86bc36304ca28ca8129a10d75115410f9239917cc2c69bbe087b3e5edfb07011 antelab/runs/errors.py +34d13e245f5a5d20ce7044a7e163ddd7526cd2df500f20de71fbff59ba53deac antelab/runs/manager.py +79dd51207502834381d92df747f1c61763ea5df0240d791634e6cd2eaee99ac4 antelab/runs/model.py +3e6bcecb226d6ee3f05600c5fcc745b9ca86acaf07ed808582ab5c14627cf2ad antelab/runs/ownership.py +4247a0c79c570bb268e0ca627cd574a1ba7e7d9f95c8bddc69547b5f109df64a antelab/runs/recovery.py +483a433f85bb74be379cdd268124b38e43b6a17d47a25cd60cbbfccd5dbc2969 antelab/runs/repository.py +26432aa19b90ca127b2f722e521d75cb55bccf5106fbb7c605139752a39b1d8f antelab/runs/sqlite_repository.py +d4e96107b58b5c663b069f4d52c8899a8d661f62943a2850a2dd7e2225536859 antelab/runs/worker.py +59b4a06a60d25eb7717006749fd1f9f4dfff1cb504ced42c0b7ab56675c2e466 pyproject.toml +8f6cf989e8f3e7e4e8879ce80926064824719093872c36bf6ee2e239d00d7bc9 uv.lock diff --git a/tests/runs/conftest.py b/tests/runs/conftest.py new file mode 100644 index 0000000..78c0f9e --- /dev/null +++ b/tests/runs/conftest.py @@ -0,0 +1,28 @@ +from collections.abc import Iterator + +import pytest + +from antelab.core.simulation import Simulation + +MAX_TEST_TICKS = 500 + + +@pytest.fixture(autouse=True) +def enforce_background_run_tick_ceiling( + monkeypatch: pytest.MonkeyPatch, +) -> Iterator[None]: + original_step = Simulation.step + steps_by_simulation: dict[int, int] = {} + + def bounded_step(simulation: Simulation): # type: ignore[no-untyped-def] + identity = id(simulation) + steps = steps_by_simulation.get(identity, 0) + 1 + if steps > MAX_TEST_TICKS: + pytest.fail( + f"background readiness test exceeded {MAX_TEST_TICKS} ticks" + ) + steps_by_simulation[identity] = steps + return original_step(simulation) + + monkeypatch.setattr(Simulation, "step", bounded_step) + yield diff --git a/tests/runs/test_adapter_contracts.py b/tests/runs/test_adapter_contracts.py new file mode 100644 index 0000000..9844e6c --- /dev/null +++ b/tests/runs/test_adapter_contracts.py @@ -0,0 +1,175 @@ +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from antelab.artifacts import RunArtifact +from antelab.artifacts.writer import write_artifact +from antelab.core.config import load_config +from antelab.core.simulation import Simulation +from antelab.experiments.runner import _metric +from antelab.runs.artifacts import ( + ArtifactRepository, + FilesystemArtifactRepository, + InMemoryArtifactRepository, +) +from antelab.runs.errors import RunStateConflictError +from antelab.runs.model import ( + ArtifactRef, + IdempotencyKey, + Lease, + RunId, + RunRequest, + RunSnapshot, +) +from antelab.runs.repository import InMemoryRunRepository, RunRepository +from antelab.runs.sqlite_repository import SqliteRunRepository + +NOW = datetime(2026, 7, 13, tzinfo=UTC) + + +def _request() -> RunRequest: + return RunRequest( + replace(load_config(Path("experiments/foraging-genesis.yaml")), ticks=3) + ) + + +def _artifact() -> RunArtifact: + simulation = Simulation.create(_request().config) + metrics: list[dict[str, int | str]] = [] + while simulation.status == "running": + result = simulation.step() + metrics.append(_metric(simulation, result.checksum)) + return RunArtifact.from_simulation(simulation, tuple(metrics)) + + +def _assert_state_conflict(operation: object) -> None: + try: + callable_operation = operation + assert callable(callable_operation) + callable_operation() + except RunStateConflictError: + return + raise AssertionError("operation accepted stale run state") + + +def assert_run_repository_contract(repository: RunRepository) -> None: + request = _request() + created = repository.create_idempotent( + request, IdempotencyKey("contract"), lambda: RunId("contract-run") + ) + repeated = repository.create_idempotent( + request, IdempotencyKey("contract"), lambda: RunId("must-not-be-used") + ) + assert repeated == created + lease = repository.claim_next( + now=NOW, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "contract-fence", + ) + assert lease is not None + renewed = repository.renew_lease( + lease, now=NOW, lease_duration=timedelta(seconds=10) + ) + progressed = repository.record_progress(renewed, tick=1, now=NOW) + assert progressed.progress.attempt_tick == 1 + stale = replace(renewed, fencing_token="stale") + _assert_state_conflict( + lambda: repository.record_progress(stale, tick=2, now=NOW) + ) + _assert_state_conflict( + lambda: repository.begin_verifying( + renewed, now=NOW + timedelta(seconds=11) + ) + ) + + +def assert_artifact_repository_contract( + repository: ArtifactRepository, tmp_path: Path +) -> None: + artifact = _artifact() + reference = repository.stage(artifact) + assert repository.exists(reference) is False + assert repository.publish(reference) == reference + assert repository.exists(reference) is True + loaded = repository.load(reference) + left = tmp_path / "left.json" + right = tmp_path / "right.json" + write_artifact(artifact, left) + write_artifact(loaded, right) + assert left.read_bytes() == right.read_bytes() + assert repository.publish(reference) == reference + + +@pytest.mark.parametrize("adapter", ["memory", "sqlite"]) +def test_run_repository_shared_contract(adapter: str, tmp_path: Path) -> None: + repository: RunRepository + if adapter == "memory": + repository = InMemoryRunRepository() + else: + repository = SqliteRunRepository(tmp_path / "runs.sqlite3") + assert_run_repository_contract(repository) + + +@pytest.mark.parametrize("adapter", ["memory", "sqlite"]) +def test_concurrent_equal_submissions_create_one_run( + adapter: str, tmp_path: Path +) -> None: + repository: RunRepository + if adapter == "memory": + repository = InMemoryRunRepository() + else: + repository = SqliteRunRepository(tmp_path / "concurrent.sqlite3") + generated = iter((RunId("run-1"), RunId("run-2"))) + + def submit(_: int) -> RunSnapshot: + return repository.create_idempotent( + _request(), IdempotencyKey("concurrent"), generated.__next__ + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + snapshots = tuple(executor.map(submit, range(2))) + + assert snapshots[0].run_id == snapshots[1].run_id == RunId("run-1") + + +@pytest.mark.parametrize("adapter", ["memory", "filesystem"]) +def test_artifact_repository_shared_contract(adapter: str, tmp_path: Path) -> None: + repository: ArtifactRepository + if adapter == "memory": + repository = InMemoryArtifactRepository() + else: + repository = FilesystemArtifactRepository(tmp_path / "artifacts") + assert_artifact_repository_contract(repository, tmp_path) + + +class _KnownBadStaleFenceRepository(InMemoryRunRepository): + def record_progress( + self, lease: Lease, *, tick: int, now: datetime + ) -> RunSnapshot: + try: + return super().record_progress(lease, tick=tick, now=now) + except RunStateConflictError: + current = self.get(lease.run_id) + return current + + +class _KnownBadVisibleStagingRepository(InMemoryArtifactRepository): + def exists(self, reference: ArtifactRef) -> bool: + return reference.artifact_id in self._staged or super().exists(reference) + + +def test_run_repository_contract_rejects_known_bad_stale_fence() -> None: + with pytest.raises(AssertionError): + assert_run_repository_contract(_KnownBadStaleFenceRepository()) + + +def test_artifact_contract_rejects_known_bad_visible_staging( + tmp_path: Path, +) -> None: + with pytest.raises(AssertionError): + assert_artifact_repository_contract( + _KnownBadVisibleStagingRepository(), tmp_path + ) diff --git a/tests/runs/test_durable_adapters.py b/tests/runs/test_durable_adapters.py new file mode 100644 index 0000000..b2a84a4 --- /dev/null +++ b/tests/runs/test_durable_adapters.py @@ -0,0 +1,212 @@ +from dataclasses import replace +from datetime import timedelta +from pathlib import Path + +import pytest + +from antelab.artifacts import RunArtifact +from antelab.core.config import load_config +from antelab.core.simulation import Simulation +from antelab.experiments.runner import _metric +from antelab.runs import RunRequest +from antelab.runs.artifacts import FilesystemArtifactRepository +from antelab.runs.errors import ArtifactIntegrityError +from antelab.runs.model import IdempotencyKey, RunId +from antelab.runs.recovery import reconcile_publications +from antelab.runs.sqlite_repository import SqliteRunRepository + + +def _request() -> RunRequest: + return RunRequest( + replace(load_config(Path("experiments/foraging-genesis.yaml")), ticks=3) + ) + + +def _artifact() -> RunArtifact: + simulation = Simulation.create(_request().config) + metrics: list[dict[str, int | str]] = [] + while simulation.status == "running": + result = simulation.step() + metrics.append(_metric(simulation, result.checksum)) + return RunArtifact.from_simulation(simulation, tuple(metrics)) + + +def test_sqlite_repository_survives_adapter_reconstruction(tmp_path: Path) -> None: + database = tmp_path / "runs.sqlite3" + first = SqliteRunRepository(database) + created = first.create_idempotent( + _request(), IdempotencyKey("durable"), lambda: RunId("run-1") + ) + + reopened = SqliteRunRepository(database) + + assert reopened.get(created.run_id) == created + assert reopened.request(created.run_id) == _request() + assert reopened.cancel(created.run_id).state == "cancelled" + assert SqliteRunRepository(database).get(created.run_id).state == "cancelled" + + +def test_filesystem_artifact_is_invisible_until_immutable_publish( + tmp_path: Path, +) -> None: + repository = FilesystemArtifactRepository(tmp_path / "artifacts") + artifact = _artifact() + + reference = repository.stage(artifact) + + assert repository.exists(reference) is False + repository.publish(reference) + assert repository.exists(reference) is True + assert repository.load(reference).to_dict() == artifact.to_dict() + + reopened = FilesystemArtifactRepository(tmp_path / "artifacts") + assert reopened.publish(reference) == reference + assert reopened.load(reference).to_dict() == artifact.to_dict() + + +def test_publish_then_restart_reconciles_completed_run_once(tmp_path: Path) -> None: + database = tmp_path / "runs.sqlite3" + repository = SqliteRunRepository(database) + artifacts = FilesystemArtifactRepository(tmp_path / "artifacts") + snapshot = repository.create_idempotent( + _request(), IdempotencyKey("crash-window"), lambda: RunId("run-1") + ) + lease = repository.claim_next( + now=snapshot.progress.updated_at, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "fence-1", + ) + assert lease is not None + verifying = repository.begin_verifying( + lease, now=snapshot.progress.updated_at + ) + lease = replace(lease, revision=verifying.revision) + reference = artifacts.stage(_artifact()) + repository.begin_publishing( + lease, + artifact_id=reference.artifact_id, + now=snapshot.progress.updated_at, + ) + artifacts.publish(reference) + + reopened_runs = SqliteRunRepository(database) + reopened_artifacts = FilesystemArtifactRepository(tmp_path / "artifacts") + reconciled = reconcile_publications(reopened_runs, reopened_artifacts) + + assert reconciled == (snapshot.run_id,) + assert reopened_runs.get(snapshot.run_id).state == "completed" + assert reconcile_publications(reopened_runs, reopened_artifacts) == () + + +def test_staged_artifact_survives_restart_and_is_reconciled(tmp_path: Path) -> None: + database = tmp_path / "runs.sqlite3" + repository = SqliteRunRepository(database) + artifacts = FilesystemArtifactRepository(tmp_path / "artifacts") + snapshot = repository.create_idempotent( + _request(), IdempotencyKey("staged-crash"), lambda: RunId("run-1") + ) + lease = repository.claim_next( + now=snapshot.progress.updated_at, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "fence-1", + ) + assert lease is not None + verifying = repository.begin_verifying( + lease, now=snapshot.progress.updated_at + ) + lease = replace(lease, revision=verifying.revision) + reference = artifacts.stage(_artifact()) + repository.begin_publishing( + lease, + artifact_id=reference.artifact_id, + now=snapshot.progress.updated_at, + ) + + reopened_runs = SqliteRunRepository(database) + reopened_artifacts = FilesystemArtifactRepository(tmp_path / "artifacts") + assert reconcile_publications(reopened_runs, reopened_artifacts) == ( + snapshot.run_id, + ) + completed = reopened_runs.get(snapshot.run_id) + assert completed.state == "completed" + assert completed.artifact == reference + assert reopened_artifacts.exists(reference) + + +def test_corrupt_staging_recovery_fails_run_terminally(tmp_path: Path) -> None: + database = tmp_path / "runs.sqlite3" + repository = SqliteRunRepository(database) + artifacts = FilesystemArtifactRepository(tmp_path / "artifacts") + snapshot = repository.create_idempotent( + _request(), IdempotencyKey("corrupt-stage"), lambda: RunId("run-1") + ) + lease = repository.claim_next( + now=snapshot.progress.updated_at, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "fence-1", + ) + assert lease is not None + verifying = repository.begin_verifying( + lease, now=snapshot.progress.updated_at + ) + lease = replace(lease, revision=verifying.revision) + reference = artifacts.stage(_artifact()) + repository.begin_publishing( + lease, + artifact_id=reference.artifact_id, + now=snapshot.progress.updated_at, + ) + staged = next((tmp_path / "artifacts/.staging").glob("*.tmp")) + staged.write_bytes(b"corrupt") + + assert reconcile_publications(repository, artifacts) == () + failed = repository.get(snapshot.run_id) + assert failed.state == "failed" + assert failed.failure is not None + assert failed.failure.code == "artifact_recovery_failed" + + +def test_corrupt_published_artifact_is_never_returned(tmp_path: Path) -> None: + repository = FilesystemArtifactRepository(tmp_path / "artifacts") + reference = repository.stage(_artifact()) + repository.publish(reference) + path = tmp_path / "artifacts" / reference.artifact_id + path.write_bytes(b"corrupt") + + with pytest.raises(ArtifactIntegrityError): + repository.load(reference) + with pytest.raises(ArtifactIntegrityError): + repository.publish(reference) + + +def test_reconciliation_rejects_artifact_from_another_request(tmp_path: Path) -> None: + database = tmp_path / "runs.sqlite3" + repository = SqliteRunRepository(database) + artifacts = FilesystemArtifactRepository(tmp_path / "artifacts") + request = RunRequest(replace(_request().config, ticks=4)) + snapshot = repository.create_idempotent( + request, IdempotencyKey("wrong-owner"), lambda: RunId("run-1") + ) + lease = repository.claim_next( + now=snapshot.progress.updated_at, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "fence-1", + ) + assert lease is not None + verifying = repository.begin_verifying( + lease, now=snapshot.progress.updated_at + ) + lease = replace(lease, revision=verifying.revision) + reference = artifacts.stage(_artifact()) + repository.begin_publishing( + lease, + artifact_id=reference.artifact_id, + now=snapshot.progress.updated_at, + ) + artifacts.publish(reference) + + assert reconcile_publications(repository, artifacts) == () + failed = repository.get(snapshot.run_id) + assert failed.state == "failed" + assert failed.failure is not None + assert failed.failure.code == "artifact_recovery_failed" diff --git a/tests/runs/test_experiment_runs.py b/tests/runs/test_experiment_runs.py new file mode 100644 index 0000000..5678118 --- /dev/null +++ b/tests/runs/test_experiment_runs.py @@ -0,0 +1,54 @@ +from dataclasses import replace +from pathlib import Path + +import pytest + +from antelab.core.config import load_config +from antelab.runs import ( + ExperimentRuns, + IdempotencyConflictError, + InMemoryArtifactRepository, + InMemoryRunRepository, + RunRequest, +) + + +def _runs() -> ExperimentRuns: + return ExperimentRuns( + InMemoryRunRepository(), + InMemoryArtifactRepository(), + id_generator=iter(("run-1", "run-2")).__next__, + ) + + +def _request(ticks: int = 10) -> RunRequest: + config = replace( + load_config(Path("experiments/foraging-genesis.yaml")), ticks=ticks + ) + return RunRequest(config) + + +def test_submit_is_idempotent_and_conflicting_reuse_is_rejected() -> None: + runs = _runs() + + first = runs.submit(_request(), idempotency_key="form-submit-1") + repeated = runs.submit(_request(), idempotency_key="form-submit-1") + + assert repeated == first + assert first.state == "queued" + assert first.progress.attempt == 0 + with pytest.raises(IdempotencyConflictError): + runs.submit(_request(11), idempotency_key="form-submit-1") + + +def test_cancel_queued_run_is_terminal_and_idempotent() -> None: + runs = _runs() + submitted = runs.submit(_request(), idempotency_key="cancel-me") + + cancelled = runs.cancel(submitted.run_id) + + assert cancelled.state == "cancelled" + assert cancelled.phase == "terminal" + assert cancelled.artifact is None + assert runs.cancel(submitted.run_id) == cancelled + assert runs.get(submitted.run_id) == cancelled diff --git a/tests/runs/test_inprocess_engine.py b/tests/runs/test_inprocess_engine.py new file mode 100644 index 0000000..79557c8 --- /dev/null +++ b/tests/runs/test_inprocess_engine.py @@ -0,0 +1,89 @@ +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from pathlib import Path +from threading import Thread +from time import monotonic + +from antelab.artifacts.writer import write_artifact +from antelab.core.config import load_config +from antelab.experiments.runner import run_experiment +from antelab.runs import ( + ExperimentRuns, + InMemoryArtifactRepository, + InMemoryRunRepository, + RunRequest, +) +from antelab.runs.worker import InProcessSimulationEngine, RunWorker + + +def test_background_short_run_matches_synchronous_reference_artifact( + tmp_path: Path, +) -> None: + config = replace( + load_config(Path("experiments/foraging-genesis.yaml")), ticks=25 + ) + reference = run_experiment( + Path("experiments/foraging-genesis.yaml"), + output=tmp_path / "reference.json", + tick_override=25, + ) + repository = InMemoryRunRepository() + artifacts = InMemoryArtifactRepository() + runs = ExperimentRuns( + repository, + artifacts, + id_generator=iter(("background-1",)).__next__, + ) + submitted = runs.submit(RunRequest(config), idempotency_key="background-short") + worker = RunWorker( + repository, + artifacts, + InProcessSimulationEngine(), + now=lambda: datetime(2026, 7, 13, tzinfo=UTC), + token_factory=lambda: "fence-1", + lease_duration=timedelta(seconds=10), + ) + + assert worker.run_once() is True + + completed = runs.get(submitted.run_id) + assert completed.state == "completed" + assert completed.progress.attempt_tick == 25 + background = runs.artifact(submitted.run_id) + write_artifact(background, tmp_path / "background.json") + assert background.to_dict() == reference.to_dict() + assert (tmp_path / "background.json").read_bytes() == ( + tmp_path / "reference.json" + ).read_bytes() + + +def test_real_inprocess_run_remains_observable_and_cancellable() -> None: + config = replace( + load_config(Path("experiments/foraging-genesis.yaml")), ticks=500 + ) + repository = InMemoryRunRepository() + artifacts = InMemoryArtifactRepository() + runs = ExperimentRuns(repository, artifacts, id_generator=lambda: "run-1") + submitted = runs.submit(RunRequest(config), idempotency_key="real-cancel") + worker = RunWorker( + repository, + artifacts, + InProcessSimulationEngine(), + now=lambda: datetime(2026, 7, 13, tzinfo=UTC), + token_factory=lambda: "fence-1", + lease_duration=timedelta(seconds=30), + ) + thread = Thread(target=worker.run_once) + thread.start() + deadline = monotonic() + 10 + observed_running = False + while monotonic() < deadline: + if runs.get(submitted.run_id).state == "running": + observed_running = True + runs.cancel(submitted.run_id) + break + thread.join(timeout=10) + + assert observed_running + assert not thread.is_alive() + assert runs.get(submitted.run_id).state == "cancelled" diff --git a/tests/runs/test_run_repository.py b/tests/runs/test_run_repository.py new file mode 100644 index 0000000..00b432f --- /dev/null +++ b/tests/runs/test_run_repository.py @@ -0,0 +1,178 @@ +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from antelab.core.config import load_config +from antelab.runs import RunRequest, RunStateConflictError +from antelab.runs.model import IdempotencyKey, RunId +from antelab.runs.repository import InMemoryRunRepository + +NOW = datetime(2026, 7, 13, tzinfo=UTC) + + +def _repository() -> tuple[InMemoryRunRepository, RunId]: + repository = InMemoryRunRepository() + config = replace( + load_config(Path("experiments/foraging-genesis.yaml")), ticks=100 + ) + snapshot = repository.create_idempotent( + RunRequest(config), + IdempotencyKey("repo-contract"), + lambda: RunId("run-1"), + ) + return repository, snapshot.run_id + + +def test_claim_is_exclusive_and_stale_fence_cannot_progress() -> None: + repository, run_id = _repository() + lease = repository.claim_next( + now=NOW, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "fence-1", + ) + + assert lease is not None + assert repository.claim_next( + now=NOW, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "fence-2", + ) is None + progressed = repository.record_progress(lease, tick=10, now=NOW) + assert progressed.progress.attempt == 1 + assert progressed.progress.attempt_tick == 10 + with pytest.raises(RunStateConflictError): + repository.record_progress( + replace(lease, fencing_token="stale"), tick=20, now=NOW + ) + + +def test_current_fence_with_stale_expected_revision_cannot_write() -> None: + repository, _ = _repository() + lease = repository.claim_next( + now=NOW, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "fence-1", + ) + assert lease is not None + progressed = repository.record_progress(lease, tick=10, now=NOW) + + with pytest.raises(RunStateConflictError, match="revision"): + repository.record_progress(lease, tick=20, now=NOW) + + current_lease = replace(lease, revision=progressed.revision) + assert repository.record_progress( + current_lease, tick=20, now=NOW + ).progress.attempt_tick == 20 + + +def test_cancel_and_completion_have_one_compare_and_swap_winner() -> None: + repository, run_id = _repository() + lease = repository.claim_next( + now=NOW, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "fence-1", + ) + assert lease is not None + + cancelling = repository.cancel(run_id) + assert cancelling.state == "cancelling" + with pytest.raises(RunStateConflictError): + repository.begin_verifying(lease, now=NOW) + assert repository.acknowledge_cancel( + replace(lease, revision=cancelling.revision), now=NOW + ).state == "cancelled" + + second_repository, second_id = _repository() + second_lease = second_repository.claim_next( + now=NOW, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "fence-2", + ) + assert second_lease is not None + verifying = second_repository.begin_verifying(second_lease, now=NOW) + assert verifying.phase == "verifying" + assert second_repository.cancel(second_id) == verifying + + +def test_cancellation_winner_cannot_be_overwritten_by_failure() -> None: + repository, run_id = _repository() + lease = repository.claim_next( + now=NOW, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "fence-1", + ) + assert lease is not None + cancelling = repository.cancel(run_id) + + with pytest.raises(RunStateConflictError): + repository.fail( + replace(lease, revision=cancelling.revision), + code="late_failure", + message="must not win", + retryable=False, + now=NOW, + ) + + +def test_expired_attempt_requeues_from_zero_without_losing_max_tick() -> None: + repository, _ = _repository() + lease = repository.claim_next( + now=NOW, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "fence-1", + ) + assert lease is not None + repository.record_progress(lease, tick=80, now=NOW) + + recovered = repository.recover_expired(now=NOW + timedelta(seconds=11)) + + assert len(recovered) == 1 + assert recovered[0].state == "queued" + assert recovered[0].progress.attempt_tick == 0 + assert recovered[0].progress.max_tick_seen == 80 + replay = repository.claim_next( + now=NOW + timedelta(seconds=11), + lease_duration=timedelta(seconds=10), + token_factory=lambda: "fence-2", + ) + assert replay is not None + assert replay.attempt == 2 + + +def test_expired_fence_cannot_verify_or_complete() -> None: + repository, _ = _repository() + lease = repository.claim_next( + now=NOW, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "expires", + ) + assert lease is not None + + with pytest.raises(RunStateConflictError, match="expired"): + repository.begin_verifying(lease, now=NOW + timedelta(seconds=11)) + + +def test_repeated_worker_loss_exhausts_recovery_budget() -> None: + repository, run_id = _repository() + now = NOW + for attempt in range(1, 4): + token = f"fence-{attempt}" + + def token_factory(token: str = token) -> str: + return token + + lease = repository.claim_next( + now=now, + lease_duration=timedelta(seconds=1), + token_factory=token_factory, + ) + assert lease is not None + now += timedelta(seconds=2) + repository.recover_expired(now=now) + + failed = repository.get(run_id) + assert failed.state == "failed" + assert failed.failure is not None + assert failed.failure.code == "worker_recovery_exhausted" diff --git a/tests/runs/test_run_worker.py b/tests/runs/test_run_worker.py new file mode 100644 index 0000000..7fc2722 --- /dev/null +++ b/tests/runs/test_run_worker.py @@ -0,0 +1,268 @@ +from collections.abc import Callable +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from antelab.artifacts import RunArtifact +from antelab.core.config import load_config +from antelab.core.simulation import Simulation +from antelab.experiments.runner import _metric +from antelab.runs import ( + ExperimentRuns, + InMemoryArtifactRepository, + InMemoryRunRepository, + RunRequest, +) +from antelab.runs.artifacts import ArtifactRepository +from antelab.runs.errors import ArtifactIntegrityError +from antelab.runs.worker import DeterministicFakeEngine, RunEngine, RunWorker + +NOW = datetime(2026, 7, 13, tzinfo=UTC) + + +def _artifact(ticks: int = 3) -> RunArtifact: + config = replace( + load_config(Path("experiments/foraging-genesis.yaml")), ticks=ticks + ) + simulation = Simulation.create(config) + metrics: list[dict[str, int | str]] = [] + while simulation.status == "running": + result = simulation.step() + metrics.append(_metric(simulation, result.checksum)) + return RunArtifact.from_simulation(simulation, tuple(metrics)) + + +def _request(ticks: int = 3) -> RunRequest: + return RunRequest( + replace(load_config(Path("experiments/foraging-genesis.yaml")), ticks=ticks) + ) + + +def _system( + engine: RunEngine, + artifacts: ArtifactRepository | None = None, +) -> tuple[ExperimentRuns, RunWorker, InMemoryRunRepository]: + repository = InMemoryRunRepository() + artifacts = artifacts or InMemoryArtifactRepository() + runs = ExperimentRuns( + repository, + artifacts, + id_generator=iter(("run-1",)).__next__, + ) + worker = RunWorker( + repository, + artifacts, + engine, + now=lambda: NOW, + token_factory=lambda: "fence-1", + lease_duration=timedelta(seconds=10), + ) + return runs, worker, repository + + +def test_worker_publishes_one_completed_artifact() -> None: + artifact = _artifact() + runs, worker, _ = _system(DeterministicFakeEngine(artifact, ticks=(1, 2, 3))) + submitted = runs.submit( + _request(), + idempotency_key="complete", + ) + + assert worker.run_once() is True + + completed = runs.get(submitted.run_id) + assert completed.state == "completed" + assert completed.progress.attempt_tick == 3 + assert completed.artifact is not None + assert runs.artifact(submitted.run_id).to_dict() == artifact.to_dict() + + +def test_running_cancel_publishes_no_artifact() -> None: + artifact = _artifact() + runs: ExperimentRuns + + def cancel_at_two(tick: int) -> None: + if tick == 2: + runs.cancel(submitted.run_id) + + engine = DeterministicFakeEngine(artifact, ticks=(1, 2, 3), on_tick=cancel_at_two) + runs, worker, _ = _system(engine) + submitted = runs.submit( + _request(), + idempotency_key="cancel", + ) + + assert worker.run_once() is True + + cancelled = runs.get(submitted.run_id) + assert cancelled.state == "cancelled" + assert cancelled.artifact is None + + +def test_unknown_engine_failure_is_permanent_and_sanitized() -> None: + class FailingEngine: + def execute( + self, request: RunRequest, progress: Callable[[int], bool] + ) -> RunArtifact: + del request, progress + raise RuntimeError("secret path /private/example") + + runs, worker, _ = _system(FailingEngine()) + submitted = runs.submit(_request(), idempotency_key="fail") + + assert worker.run_once() is True + + failed = runs.get(submitted.run_id) + assert failed.state == "failed" + assert failed.failure is not None + assert failed.failure.code == "engine_failure" + assert "private" not in failed.failure.message + + +def test_cancel_winner_is_not_overwritten_by_late_engine_error() -> None: + runs: ExperimentRuns + + class CancelThenFailEngine: + def execute( + self, request: RunRequest, progress: Callable[[int], bool] + ) -> RunArtifact: + del request, progress + runs.cancel(submitted.run_id) + raise RuntimeError("engine failed after cancellation won") + + runs, worker, _ = _system(CancelThenFailEngine()) + submitted = runs.submit(_request(), idempotency_key="cancel-then-error") + + assert worker.run_once() is True + assert runs.get(submitted.run_id).state == "cancelled" + + +def test_cancel_winner_beats_late_artifact_ownership_error() -> None: + runs: ExperimentRuns + + class CancelThenReturnWrongArtifact: + def execute( + self, request: RunRequest, progress: Callable[[int], bool] + ) -> RunArtifact: + del request, progress + runs.cancel(submitted.run_id) + return _artifact(ticks=4) + + runs, worker, _ = _system(CancelThenReturnWrongArtifact()) + submitted = runs.submit(_request(), idempotency_key="cancel-wrong-artifact") + + assert worker.run_once() is True + assert runs.get(submitted.run_id).state == "cancelled" + + +def test_transient_stage_io_waits_for_lease_recovery() -> None: + class TransientStageArtifacts(InMemoryArtifactRepository): + def stage(self, artifact: RunArtifact): # type: ignore[no-untyped-def] + del artifact + raise OSError("temporary staging I/O") + + runs, worker, repository = _system( + DeterministicFakeEngine(_artifact(), ticks=(1, 2, 3)), + TransientStageArtifacts(), + ) + submitted = runs.submit(_request(), idempotency_key="transient-stage") + + assert worker.run_once() is True + waiting = runs.get(submitted.run_id) + assert waiting.state == "running" + assert waiting.phase == "verifying" + recovered = repository.recover_expired( + now=NOW + timedelta(seconds=11) + ) + assert recovered[0].state == "queued" + + +def test_transient_publish_io_preserves_staging_recovery_state() -> None: + class TransientPublishArtifacts(InMemoryArtifactRepository): + def publish(self, reference): # type: ignore[no-untyped-def] + del reference + raise OSError("temporary publication I/O") + + runs, worker, _ = _system( + DeterministicFakeEngine(_artifact(), ticks=(1, 2, 3)), + TransientPublishArtifacts(), + ) + submitted = runs.submit(_request(), idempotency_key="transient-publish") + + assert worker.run_once() is True + waiting = runs.get(submitted.run_id) + assert waiting.state == "running" + assert waiting.phase == "publishing" + assert waiting.failure is None + + +def test_permanent_publish_integrity_error_fails_safely() -> None: + class InvalidPublishArtifacts(InMemoryArtifactRepository): + def publish(self, reference): # type: ignore[no-untyped-def] + del reference + raise ArtifactIntegrityError("secret invalid bytes") + + runs, worker, _ = _system( + DeterministicFakeEngine(_artifact(), ticks=(1, 2, 3)), + InvalidPublishArtifacts(), + ) + submitted = runs.submit(_request(), idempotency_key="invalid-publish") + + assert worker.run_once() is True + failed = runs.get(submitted.run_id) + assert failed.state == "failed" + assert failed.failure is not None + assert failed.failure.code == "artifact_publication_failed" + assert "secret" not in failed.failure.message + + +def test_progress_persistence_is_bounded_to_cadence_and_terminal_tick() -> None: + class RecordingRepository(InMemoryRunRepository): + def __init__(self) -> None: + super().__init__() + self.persisted_ticks: list[int] = [] + + def record_progress(self, lease, *, tick, now): # type: ignore[no-untyped-def] + self.persisted_ticks.append(tick) + return super().record_progress(lease, tick=tick, now=now) + + repository = RecordingRepository() + artifacts = InMemoryArtifactRepository() + runs = ExperimentRuns(repository, artifacts, id_generator=lambda: "run-1") + worker = RunWorker( + repository, + artifacts, + DeterministicFakeEngine(_artifact(25), ticks=range(1, 26)), + now=lambda: NOW, + token_factory=lambda: "fence-1", + lease_duration=timedelta(seconds=10), + ) + runs.submit(_request(25), idempotency_key="cadence") + + assert worker.run_once() is True + assert repository.persisted_ticks == [10, 20, 25] + + +def test_transient_repository_io_is_not_permanent_engine_failure() -> None: + class TransientProgressRepository(InMemoryRunRepository): + def record_progress(self, lease, *, tick, now): # type: ignore[no-untyped-def] + del lease, tick, now + raise OSError("temporary repository I/O") + + repository = TransientProgressRepository() + artifacts = InMemoryArtifactRepository() + runs = ExperimentRuns(repository, artifacts, id_generator=lambda: "run-1") + worker = RunWorker( + repository, + artifacts, + DeterministicFakeEngine(_artifact(10), ticks=range(1, 11)), + now=lambda: NOW, + token_factory=lambda: "fence-1", + lease_duration=timedelta(seconds=10), + ) + submitted = runs.submit(_request(10), idempotency_key="transient-repository") + + assert worker.run_once() is True + waiting = runs.get(submitted.run_id) + assert waiting.state == "running" + assert waiting.failure is None diff --git a/tests/test_background_runs_verifier.py b/tests/test_background_runs_verifier.py new file mode 100644 index 0000000..ff953da --- /dev/null +++ b/tests/test_background_runs_verifier.py @@ -0,0 +1,86 @@ +import shutil +from pathlib import Path + +from scripts.validate_background_runs_readiness import ( + run_behavioral_contract, + validate_round_one_scope, + validate_structure, +) + + +def test_background_runs_verifier_accepts_current_known_good() -> None: + assert validate_structure(Path.cwd()) == () + assert run_behavioral_contract(Path.cwd()) == 0 + + +def test_background_runs_verifier_rejects_missing_locked_scenario( + tmp_path: Path, +) -> None: + root = Path.cwd() + for relative in ( + "antelab/runs/model.py", + "antelab/runs/manager.py", + "antelab/runs/repository.py", + "antelab/runs/sqlite_repository.py", + "antelab/runs/artifacts.py", + "antelab/runs/worker.py", + "antelab/runs/recovery.py", + ): + target = tmp_path / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text((root / relative).read_text(encoding="utf-8"), encoding="utf-8") + tests_target = tmp_path / "tests/runs/test_contract.py" + tests_target.parent.mkdir(parents=True, exist_ok=True) + tests_target.write_text( + "\n".join( + path.read_text(encoding="utf-8") + for path in sorted((root / "tests/runs").glob("test_*.py")) + ), + encoding="utf-8", + ) + manifest = tmp_path / "tests/manifests/background-run-scenarios.txt" + manifest.parent.mkdir(parents=True, exist_ok=True) + lines = (root / manifest.relative_to(tmp_path)).read_text(encoding="utf-8").splitlines() + manifest.write_text("\n".join(lines[1:]) + "\n", encoding="utf-8") + + assert "scenario manifest differs" in " ".join(validate_structure(tmp_path)) + + +def test_background_runs_verifier_rejects_round_one_product_mutation( + tmp_path: Path, +) -> None: + root = Path.cwd() + for relative in ( + "antelab/runs", + "antelab/cli.py", + "antelab/experiments/runner.py", + "pyproject.toml", + "uv.lock", + "tests/manifests", + ): + source = root / relative + target = tmp_path / relative + target.parent.mkdir(parents=True, exist_ok=True) + if source.is_dir(): + shutil.copytree(source, target) + else: + shutil.copy2(source, target) + with (tmp_path / "antelab/runs/model.py").open("a", encoding="utf-8") as stream: + stream.write("\n# simulated unauthorized Round 1 mutation\n") + + assert ( + "Round 1 protected file changed: antelab/runs/model.py" + in validate_round_one_scope(tmp_path) + ) + + +def test_background_runs_verifier_rejects_added_product_file(tmp_path: Path) -> None: + root = Path.cwd() + shutil.copytree(root / "antelab", tmp_path / "antelab") + shutil.copytree(root / "tests/manifests", tmp_path / "tests/manifests") + shutil.copy2(root / "pyproject.toml", tmp_path / "pyproject.toml") + shutil.copy2(root / "uv.lock", tmp_path / "uv.lock") + added = tmp_path / "antelab/runs/unauthorized.py" + added.write_text("VALUE = 'unauthorized'\n", encoding="utf-8") + + assert "Round 1 product file set changed" in validate_round_one_scope(tmp_path) From d24470505b26bab3cdb0bdfba10276f57aed0862 Mon Sep 17 00:00:00 2001 From: Merengues0x2A Date: Tue, 14 Jul 2026 09:10:07 +0800 Subject: [PATCH 2/4] chore: reconcile v1 roadmap controller --- controllers/antelab-v1-roadmap.controller.md | 42 +++++++---- .../antelab-v1-roadmap/round-03.md | 42 +++++++++++ state/antelab-v1-roadmap.STATE.md | 69 +++++++++++-------- 3 files changed, 112 insertions(+), 41 deletions(-) create mode 100644 reports/controllers/antelab-v1-roadmap/round-03.md diff --git a/controllers/antelab-v1-roadmap.controller.md b/controllers/antelab-v1-roadmap.controller.md index 8399c1f..a70dd4f 100644 --- a/controllers/antelab-v1-roadmap.controller.md +++ b/controllers/antelab-v1-roadmap.controller.md @@ -12,7 +12,7 @@ into one autonomous mutator loop. - Execution authority: run-to-stop. - Concurrency: one active queue item. -- Worktree: `/private/tmp/antelab-digital-evolution-kernel`. +- Worktree: `/private/tmp/antelab-v1-performance-gate-worktree`. - Protected checkout: `/Users/zec/Documents/Repos/AnteLab`; read-only and bound to the recorded HEAD and WIP fingerprint. - Commit, push, merge, release, deployment, deletion, and external communication @@ -21,18 +21,22 @@ into one autonomous mutator loop. ## Queue -1. `phase-a-safe-landing` — active. Audit the reset boundary, bind remote CI - evidence, obtain an independent checker verdict, and stop before merge. -2. `public-v1-performance-gate` — blocked on Queue 1 landing. -3. `phase-b-falsifiable-evolution-proof` — blocked on Queue 2 and design approval. +1. `phase-a-safe-landing` — succeeded at remote main `531c390` with both main + CI jobs passing. +2. `background-experiment-runs-landing` — waiting-human. The human-approved + product contract treats about 100 seconds as valuable background execution; + 30 seconds is non-blocking stretch telemetry. Local implementation commit + `38e7b84` is verified but not pushed and has no PR or remote CI. +3. `phase-b-falsifiable-evolution-proof` — blocked on Queue 2 landing and design + approval. 4. `phase-c-live-first-observer` — blocked on Phase B gate and architecture approval. 5. `phase-d-release-and-adoption` — human-led; blocked on all prior gates. -## Queue 1 Locked Verification +## Queue 1 Landing Evidence -- Feature branch and committed HEAD match the approved ref. -- PR is open and mergeable; base is `main`; head and diff totals match. -- Required Linux x86_64 and macOS arm64 checks have completed successfully. +- PR #1 merged as `531c390de1c7d7b70decb728c6489c80070b94e3`. +- Required Linux x86_64 and macOS arm64 checks completed successfully on the PR + head and on the main push run `29191353730`. - Every one of the 314 committed diff paths is classified against design section 15 and the locked Phase A structure; unclassified count is zero. - The 412,060 textual deletions split exactly into 410,201 lines on approved @@ -43,17 +47,26 @@ into one autonomous mutator loop. energy accounting, packaging, and CI. - The protected main checkout HEAD and WIP fingerprint remain unchanged. +## Queue 2 Landing Gate + +- Local branch: `feature/v1-performance-gate` at `38e7b849bc5e27d43f70c0e73ff9056a560c2f8a`. +- Locked local evidence: readiness 36 / 36; Ruff PASS; strict mypy PASS; isolated + pytest 590 / 590 under the global 500-step guard; Standards and Spec checker + PASS. +- Push, PR creation, merge, default-runner switching, 5,000-tick execution, and + cleanup remain separate human gates. + ## Progress and Stop Rules -- Progress metric: reviewed committed files / 314; required checks successful / 2. +- Progress metric: current queue item remote checks successful / 2 and unresolved + landing gates. - Failure fingerprint: normalized set of failed locked assertions and checker findings. - Stop `stopped-no-improvement` after two rounds with the same fingerprint and zero positive progress delta. - Stop `waiting-human` at any irreversible action or when refs, PR state, CI, or protected WIP differs from durable state. -- Queue 1 success means merge-ready evidence exists. It does not authorize merge. -- Do not activate Queue 2 until remote `main` landing, post-merge CI, clean wheel - installation, and CLI smoke have been proved. +- Queue 1 is complete. Queue 2 remains `waiting-human` until its branch is + published, a PR is created, and remote CI is proven. ## Durable Artifacts @@ -61,3 +74,6 @@ into one autonomous mutator loop. - Controller reports: `reports/controllers/antelab-v1-roadmap/round-NN.md`. - Queue 1 audit verifier: `scripts/controllers/audit_phase_a_reset.py`. - Queue 1 manifest: `reports/controllers/antelab-v1-roadmap/reset-boundary-manifest.tsv`. +- Queue 2 child contracts: `goals/antelab-background-experiment-runs.goal.md`, + `goals/background-runs-readiness-repair.goal.md`, and + `goals/background-runs-precommit-closure.goal.md`. diff --git a/reports/controllers/antelab-v1-roadmap/round-03.md b/reports/controllers/antelab-v1-roadmap/round-03.md new file mode 100644 index 0000000..663489f --- /dev/null +++ b/reports/controllers/antelab-v1-roadmap/round-03.md @@ -0,0 +1,42 @@ +# AnteLab V1 Roadmap Controller — Round 03 + +Verdict: `waiting-human` + +## Scope + +Authority reconciliation only. No product, verifier, dependency, CLI, default +runner, remote ref, PR, merge, deployment, or cleanup mutation. + +## Revalidated Evidence + +- Original Phase A worktree is absent and registered as prunable. +- PR #1 is MERGED at `531c390de1c7d7b70decb728c6489c80070b94e3`. +- PR checks `kernel-x86_64` and `kernel-arm64` are SUCCESS. +- Main push CI run `29191353730` is SUCCESS at exact head `531c390`; both platform + jobs passed Ruff, mypy, pytest, 500-tick golden repeatability, wheel install, + and CLI smoke. +- Protected checkout remains at `23060035` with unchanged WIP fingerprint + `92fd8d834df84b045e425de483ed535814a3d2f5c5085c5444004f70022bcf3d`. +- Current isolated worktree is clean at local commit `38e7b84` before this + controller-only reconciliation. +- `origin/feature/v1-performance-gate` does not exist. + +## Progress Delta + +- Queue 1: advanced from stale `running` evidence to proven `succeeded`. +- Active identity: rebound from missing Phase A worktree to the verified Queue 2 + worktree and commit. +- Queue 2 remote checks remain 0 / 2 because push/PR authority is absent. + +## Failure and Stop + +- Fingerprint: `locked_identity_changed_after_phase_a_landing`. +- Classification: external/ref change and permission gate, not product failure. +- Stop: `waiting-human` before commit, push, PR creation, merge, default switch, + 5,000-tick execution, or cleanup. + +## Unique Next Focus + +After explicit authorization, create one scoped Controller reconciliation +commit, push `feature/v1-performance-gate`, create a PR to `main` without +merging, and verify both remote CI jobs. diff --git a/state/antelab-v1-roadmap.STATE.md b/state/antelab-v1-roadmap.STATE.md index d533be4..5fb39f9 100644 --- a/state/antelab-v1-roadmap.STATE.md +++ b/state/antelab-v1-roadmap.STATE.md @@ -1,51 +1,64 @@ # State: AnteLab V1 Roadmap -Schema version: 2 +Schema version: 3 ## Controller Status -- Status: `running`. -- Active queue item: `phase-a-safe-landing`. -- Active round: 2. +- Status: `waiting-human`. +- Active queue item: `background-experiment-runs-landing`. +- Active round: 3. - Concurrency: 1. - Next queue items: blocked. ## Locked Identities -- Worktree: `/private/tmp/antelab-digital-evolution-kernel`. -- Branch: `feature/digital-evolution-kernel`. -- Committed HEAD: `83c9d35c137dd85a40e866d8a94b26d8ef4a37bc`. -- Merge base / remote main: `714f999a07f0a368e2add4e6a3dc3b9b0e0f455c`. -- PR: `https://github.com/powerball0x2a/AnteLab/pull/1`. +- Worktree: `/private/tmp/antelab-v1-performance-gate-worktree`. +- Branch: `feature/v1-performance-gate`. +- Committed HEAD: `38e7b849bc5e27d43f70c0e73ff9056a560c2f8a`. +- Remote main: `531c390de1c7d7b70decb728c6489c80070b94e3`. +- Remote feature branch: absent at reconciliation time. +- Phase A PR: `https://github.com/powerball0x2a/AnteLab/pull/1`. - Protected checkout HEAD: `23060035f5e2547443f28951a251c92fc6688aab`. -- Protected checkout WIP SHA-256: `92fd8d834df84b045e425de483ed535814a3d2f5c5085c5444004f70022bcf3d`. +- Protected checkout WIP SHA-256: + `92fd8d834df84b045e425de483ed535814a3d2f5c5085c5444004f70022bcf3d`. ## Current Evidence -- PR is `OPEN` and `MERGEABLE`; base `main`; head matches the locked commit. -- Committed PR diff: 34 commits, 314 files, 12,261 additions, 412,060 deletions. -- `kernel-x86_64`: completed `SUCCESS`. -- `kernel-arm64`: completed `SUCCESS`. -- Reset-boundary audit: passed; manifest records every committed path. -- Independent checker: post-repair `PASS`; all four initial findings closed. +- PR #1 is `MERGED`; merge commit `531c390`; merged at + `2026-07-12T11:43:59Z`. +- PR head checks: `kernel-x86_64` SUCCESS and `kernel-arm64` SUCCESS. +- Main push run `29191353730`: SUCCESS at exact head `531c390`; both kernel jobs, + Ruff, mypy, pytest, 500-tick golden repeatability, wheel install, and CLI smoke + succeeded. +- Phase A reset-boundary audit remains 314 / 314 classified with independent + checker PASS. +- Background Experiment Runs local commit `38e7b84`: readiness 36 / 36, Ruff + PASS, strict mypy PASS, isolated pytest 590 / 590 under a 500-step guard, and + independent Standards/Spec PASS. +- Synchronous CLI/default runner remains unchanged; no 5,000-tick execution was + performed for the background implementation. ## Progress -- Reviewed committed files: 314 / 314. -- Unclassified committed files: 0. -- Required remote checks on baseline PR head `83c9d35`: 2 / 2 successful. -- Required remote checks on the eventual repair commit: 0 / 2; no repair commit - exists and no push is authorized. +- Queue 1: succeeded. +- Queue 2 local implementation readiness: succeeded. +- Queue 2 remote publication/CI: 0 / 2 required platform checks; no remote branch + or PR exists. - Consecutive no-improvement rounds: 0 / 2. -## Mutation Boundary +## Current Gate -Queue 1 may change only controller/evidence assets and correct evidence defects. -Product code, verifier thresholds, performance contracts, merge state, and Queue -2 work are outside this round. +- Failure fingerprint: `locked_identity_changed_after_phase_a_landing`. +- Classification: expected external/ref change plus missing permission. +- Human authority required for a scoped Controller reconciliation commit, push + of `feature/v1-performance-gate`, and PR creation to `main`. +- Merge, default-runner switch, 5,000 ticks, deployment, publication, and cleanup + remain unauthorized. ## Next Focus -Create and push the explicitly authorized scoped verifier/evidence repair -commit. Then rebind PR head/diff and require both remote CI jobs to pass before -requesting a separate merge strategy and authorization. +Obtain explicit authorization to commit this Controller reconciliation, push the +feature branch, and create a non-merging PR to `main` solely to trigger and +verify remote CI. + +Updated: `2026-07-14`, writer `Roadmap Controller`. From f57da5418ca42de5d66dba35fcc83ec9ccd71b30 Mon Sep 17 00:00:00 2001 From: Merengues0x2A Date: Tue, 14 Jul 2026 14:06:15 +0800 Subject: [PATCH 3/4] fix: close background run pre-landing blockers --- RUNNING.md | 4 +- antelab/runs/artifacts.py | 48 ++++- antelab/runs/model.py | 19 +- antelab/runs/ownership.py | 4 +- antelab/runs/repository.py | 2 + antelab/runs/sqlite_repository.py | 67 +++++-- antelab/runs/worker.py | 83 ++++++--- .../background-runs-prelanding-repair.goal.md | 81 +++++++++ ...kground-runs-prelanding-repair-round-01.md | 52 ++++++ ...kground-runs-prelanding-repair-round-02.md | 34 ++++ ...kground-runs-prelanding-repair-round-03.md | 69 +++++++ scripts/validate_background_runs_readiness.py | 95 +++++++++- ...background-runs-prelanding-repair.STATE.md | 65 +++++++ .../background-run-scenario-tests.txt | 30 +++ tests/manifests/background-run-scenarios.txt | 7 + ...ackground-runs-prelanding-round-one.sha256 | 32 ++++ tests/runs/test_adapter_contracts.py | 171 ++++++++++++++++++ tests/runs/test_durable_adapters.py | 101 ++++++++++- tests/runs/test_experiment_runs.py | 34 +++- tests/runs/test_inprocess_engine.py | 31 ++++ tests/runs/test_run_worker.py | 47 +++++ tests/test_background_runs_verifier.py | 44 +++++ 22 files changed, 1064 insertions(+), 56 deletions(-) create mode 100644 goals/background-runs-prelanding-repair.goal.md create mode 100644 reports/goals/background-runs-prelanding-repair-round-01.md create mode 100644 reports/goals/background-runs-prelanding-repair-round-02.md create mode 100644 reports/goals/background-runs-prelanding-repair-round-03.md create mode 100644 state/background-runs-prelanding-repair.STATE.md create mode 100644 tests/manifests/background-run-scenario-tests.txt create mode 100644 tests/manifests/background-runs-prelanding-round-one.sha256 diff --git a/RUNNING.md b/RUNNING.md index 54bceb9..5ddb43e 100644 --- a/RUNNING.md +++ b/RUNNING.md @@ -33,5 +33,5 @@ bash scripts/loops/verify-antelab-kernel-closure.sh ``` The local closure budget is 5,000 ticks in at most 120 seconds and 512 MiB peak -RSS. The stricter 30-second target remains a public V1 release gate; Phase A -does not revise or claim to meet it. +RSS. The 30-second target is non-blocking stretch telemetry: record it, but do +not fail a run or block release solely because it exceeds 30 seconds. diff --git a/antelab/runs/artifacts.py b/antelab/runs/artifacts.py index a5465f0..840aa4d 100644 --- a/antelab/runs/artifacts.py +++ b/antelab/runs/artifacts.py @@ -130,6 +130,7 @@ def publish(self, reference: ArtifactRef) -> ArtifactRef: final = self._path(reference) if final.exists(): self._verify_bytes(final.read_bytes(), reference) + self._detach_published_aliases(final, reference.sha256) return reference staged = self._staged.get(reference.artifact_id) if staged is None or not staged.exists(): @@ -140,8 +141,16 @@ def publish(self, reference: ArtifactRef) -> ArtifactRef: payload = staged.read_bytes() self._verify_bytes(payload, reference) final.parent.mkdir(parents=True, exist_ok=True) + publishing = final.parent / f".{reference.sha256}.{uuid4().hex}.publishing" + with publishing.open("xb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + publishing.chmod(0o444) + linked = False try: - final.hardlink_to(staged) + final.hardlink_to(publishing) + linked = True except FileExistsError: self._verify_bytes(final.read_bytes(), reference) directory_fd = os.open(final.parent, os.O_RDONLY) @@ -149,8 +158,45 @@ def publish(self, reference: ArtifactRef) -> ArtifactRef: os.fsync(directory_fd) finally: os.close(directory_fd) + if linked: + self._detach_published_aliases(final, reference.sha256) + final.chmod(0o644) + staged.unlink() + self._staged.pop(reference.artifact_id, None) + staging_fd = os.open(self._staging, os.O_RDONLY) + try: + os.fsync(staging_fd) + finally: + os.close(staging_fd) + else: + publishing.unlink() + directory_fd = os.open(final.parent, os.O_RDONLY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) return reference + def _detach_published_aliases(self, final: Path, digest: str) -> None: + candidates = ( + *self._staging.glob(f"{digest}.*.tmp"), + *final.parent.glob(f".{digest}.*.publishing"), + ) + changed_directories: set[Path] = set() + for candidate in candidates: + try: + if candidate.samefile(final): + candidate.unlink() + changed_directories.add(candidate.parent) + except FileNotFoundError: + continue + for directory in changed_directories: + directory_fd = os.open(directory, os.O_RDONLY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + def exists(self, reference: ArtifactRef) -> bool: final = self._path(reference) if not final.exists(): diff --git a/antelab/runs/model.py b/antelab/runs/model.py index 706efa6..a687875 100644 --- a/antelab/runs/model.py +++ b/antelab/runs/model.py @@ -3,7 +3,7 @@ from __future__ import annotations import json -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import UTC, datetime from hashlib import sha256 from typing import Literal, NewType @@ -22,15 +22,26 @@ @dataclass(frozen=True) class RunRequest: config: SimulationConfig + engine_version: str = field(default_factory=lambda: __version__) + artifact_schema_version: int = 1 def __post_init__(self) -> None: - self.config.validate() + normalized = SimulationConfig.from_mapping(self.config.to_dict()) + normalized.validate() + object.__setattr__(self, "config", normalized) + if type(self.engine_version) is not str or not self.engine_version: + raise ValueError("engine version must be non-empty") + if ( + type(self.artifact_schema_version) is not int + or self.artifact_schema_version != 1 + ): + raise ValueError("artifact schema version must equal 1") def digest(self) -> str: payload = { - "artifact_schema_version": 1, + "artifact_schema_version": self.artifact_schema_version, "config": self.config.to_dict(), - "engine_version": __version__, + "engine_version": self.engine_version, } encoded = json.dumps( payload, sort_keys=True, separators=(",", ":"), allow_nan=False diff --git a/antelab/runs/ownership.py b/antelab/runs/ownership.py index a32b189..f3efcb5 100644 --- a/antelab/runs/ownership.py +++ b/antelab/runs/ownership.py @@ -7,7 +7,9 @@ def validate_artifact_ownership(request: RunRequest, artifact: RunArtifact) -> None: artifact.validate() if ( - artifact.config != request.config.to_dict() + artifact.schema_version != request.artifact_schema_version + or artifact.engine_version != request.engine_version + or artifact.config != request.config.to_dict() or artifact.ticks_requested != request.config.ticks or artifact.seed != request.config.seed ): diff --git a/antelab/runs/repository.py b/antelab/runs/repository.py index 114177c..3225b2b 100644 --- a/antelab/runs/repository.py +++ b/antelab/runs/repository.py @@ -111,6 +111,8 @@ def create_idempotent( ) return existing.snapshot run_id = run_id_factory() + if run_id in self._records: + raise RunStateConflictError("run id is already in use") snapshot = initial_snapshot(run_id, request) self._records[run_id] = _RunRecord( snapshot=snapshot, diff --git a/antelab/runs/sqlite_repository.py b/antelab/runs/sqlite_repository.py index 348da4e..f71a0d0 100644 --- a/antelab/runs/sqlite_repository.py +++ b/antelab/runs/sqlite_repository.py @@ -10,6 +10,7 @@ from threading import RLock from typing import TypeVar, cast +from antelab import __version__ from antelab.core.config import SimulationConfig from antelab.runs.model import ( ArtifactRef, @@ -27,6 +28,7 @@ from antelab.runs.repository import InMemoryRunRepository Result = TypeVar("Result") +DURABLE_SCHEMA_VERSION = 2 class SqliteRunRepository(InMemoryRunRepository): @@ -42,7 +44,7 @@ def __init__(self, path: Path) -> None: ) connection.execute( "INSERT OR IGNORE INTO run_state(singleton, payload) " - "VALUES (1, '{\"schema_version\":1,\"runs\":{}}')" + "VALUES (1, '{\"schema_version\":2,\"runs\":{}}')" ) self._read(lambda: None) @@ -187,28 +189,36 @@ def _connect(self) -> sqlite3.Connection: return connection def _read(self, operation: Callable[[], Result]) -> Result: - with self._sqlite_lock, self._connect() as connection: - self._load(connection) - return operation() + try: + with self._sqlite_lock, self._connect() as connection: + self._load(connection) + return operation() + except sqlite3.OperationalError as error: + _raise_transient_sqlite_io(error) + raise def _write(self, operation: Callable[[], Result]) -> Result: - with self._sqlite_lock, self._connect() as connection: - connection.execute("BEGIN IMMEDIATE") - self._load(connection) - result = operation() - connection.execute( - "UPDATE run_state SET payload = ? WHERE singleton = 1", - (self._dump(),), - ) - connection.commit() - return result + try: + with self._sqlite_lock, self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + self._load(connection) + result = operation() + connection.execute( + "UPDATE run_state SET payload = ? WHERE singleton = 1", + (self._dump(),), + ) + connection.commit() + return result + except sqlite3.OperationalError as error: + _raise_transient_sqlite_io(error) + raise def _load(self, connection: sqlite3.Connection) -> None: row = connection.execute( "SELECT payload FROM run_state WHERE singleton = 1" ).fetchone() envelope = cast(dict[str, object], json.loads(cast(str, row[0]))) - if envelope.get("schema_version") != 1 or set(envelope) != { + if envelope.get("schema_version") not in {1, DURABLE_SCHEMA_VERSION} or set(envelope) != { "schema_version", "runs", }: @@ -222,13 +232,22 @@ def _load(self, connection: sqlite3.Connection) -> None: request = RunRequest( SimulationConfig.from_mapping( cast(dict[str, object], record_raw["config"]) - ) + ), + engine_version=cast( + str, record_raw.get("engine_version", __version__) + ), + artifact_schema_version=cast( + int, record_raw.get("artifact_schema_version", 1) + ), ) key = IdempotencyKey(cast(str, record_raw["idempotency_key"])) + snapshot = _snapshot_from_dict( + cast(dict[str, object], record_raw["snapshot"]) + ) + if request.digest() != snapshot.request_digest: + raise ValueError("durable Run Request binding differs from snapshot") record = _RunRecord( - snapshot=_snapshot_from_dict( - cast(dict[str, object], record_raw["snapshot"]) - ), + snapshot=snapshot, request=request, idempotency_key=key, lease=_lease_from_dict( @@ -249,6 +268,8 @@ def _dump(self) -> str: str(run_id): { "candidate_artifact_id": record.candidate_artifact_id, "config": record.request.config.to_dict(), + "engine_version": record.request.engine_version, + "artifact_schema_version": record.request.artifact_schema_version, "idempotency_key": str(record.idempotency_key), "lease": _lease_to_dict(record.lease), "recovery_count": record.recovery_count, @@ -257,12 +278,18 @@ def _dump(self) -> str: for run_id, record in self._records.items() } return json.dumps( - {"schema_version": 1, "runs": raw}, + {"schema_version": DURABLE_SCHEMA_VERSION, "runs": raw}, sort_keys=True, separators=(",", ":"), ) +def _raise_transient_sqlite_io(error: sqlite3.OperationalError) -> None: + message = str(error).lower() + if "locked" in message or "busy" in message: + raise OSError("run repository is temporarily unavailable") from error + + def _snapshot_to_dict(snapshot: RunSnapshot) -> dict[str, object]: return { "artifact": None diff --git a/antelab/runs/worker.py b/antelab/runs/worker.py index 1169f39..a50bdea 100644 --- a/antelab/runs/worker.py +++ b/antelab/runs/worker.py @@ -94,14 +94,18 @@ def __init__( self._lease_duration = lease_duration def run_once(self) -> bool: - lease = self._repository.claim_next( - now=self._now(), - lease_duration=self._lease_duration, - token_factory=self._token_factory, - ) + try: + lease = self._repository.claim_next( + now=self._now(), + lease_duration=self._lease_duration, + token_factory=self._token_factory, + ) + except OSError: + return True if lease is None: return False active_lease = lease + last_persisted_at = lease.expires_at - self._lease_duration try: request = self._repository.request(active_lease.run_id) except OSError: @@ -135,6 +139,19 @@ def fail_or_cancel( active_lease, now=self._now() ) + def persist_progress(tick: int, *, now: datetime) -> None: + nonlocal active_lease, last_persisted_at + active_lease = self._repository.renew_lease( + active_lease, + now=now, + lease_duration=self._lease_duration, + ) + progressed = self._repository.record_progress( + active_lease, tick=tick, now=now + ) + active_lease = replace(active_lease, revision=progressed.revision) + last_persisted_at = now + def progress(tick: int) -> bool: nonlocal active_lease try: @@ -143,20 +160,12 @@ def progress(tick: int) -> bool: raise _TransientRepositoryIOError from error if snapshot.state == "cancelling": return False - if tick % 10 != 0 and tick != request.config.ticks: + now = self._now() + time_due = now - last_persisted_at >= timedelta(seconds=1) + if tick % 10 != 0 and tick != request.config.ticks and not time_due: return True try: - active_lease = self._repository.renew_lease( - active_lease, - now=self._now(), - lease_duration=self._lease_duration, - ) - progressed = self._repository.record_progress( - active_lease, tick=tick, now=self._now() - ) - active_lease = replace( - active_lease, revision=progressed.revision - ) + persist_progress(tick, now=now) except OSError as error: raise _TransientRepositoryIOError from error except RunStateConflictError: @@ -194,6 +203,22 @@ def progress(tick: int) -> bool: ) return True + try: + current = self._repository.get(active_lease.run_id) + if artifact.ticks_completed > current.progress.attempt_tick: + persist_progress(artifact.ticks_completed, now=self._now()) + except OSError: + return True + except RunStateConflictError: + if self._repository.get(active_lease.run_id).state == "cancelling": + current = self._repository.get(active_lease.run_id) + active_lease = replace(active_lease, revision=current.revision) + self._repository.acknowledge_cancel( + active_lease, now=self._now() + ) + return True + raise + try: verifying = self._repository.begin_verifying( active_lease, now=self._now() @@ -206,6 +231,8 @@ def progress(tick: int) -> bool: self._repository.acknowledge_cancel(active_lease, now=self._now()) return True raise + except OSError: + return True try: reference = self._artifacts.stage(artifact) except OSError: @@ -217,11 +244,14 @@ def progress(tick: int) -> bool: retryable=False, ) return True - publishing = self._repository.begin_publishing( - active_lease, - artifact_id=reference.artifact_id, - now=self._now(), - ) + try: + publishing = self._repository.begin_publishing( + active_lease, + artifact_id=reference.artifact_id, + now=self._now(), + ) + except OSError: + return True active_lease = replace(active_lease, revision=publishing.revision) try: published = self._artifacts.publish(reference) @@ -234,7 +264,10 @@ def progress(tick: int) -> bool: retryable=False, ) return True - self._repository.complete( - active_lease, artifact=published, now=self._now() - ) + try: + self._repository.complete( + active_lease, artifact=published, now=self._now() + ) + except OSError: + return True return True diff --git a/goals/background-runs-prelanding-repair.goal.md b/goals/background-runs-prelanding-repair.goal.md new file mode 100644 index 0000000..c42022d --- /dev/null +++ b/goals/background-runs-prelanding-repair.goal.md @@ -0,0 +1,81 @@ +# Goal: Background Runs Pre-Landing Repair + +Status: `succeeded` + +- Control shape: bounded goal with three heterogeneous rounds; concurrency one. +- Authority: run-to-stop, maximum 3 rounds / 4 hours. +- Workspace: `/private/tmp/antelab-v1-performance-gate-worktree`. +- Exact start ref: `d24470505b26bab3cdb0bdfba10276f57aed0862` on + `feature/v1-performance-gate`. +- Canonical contract: + `docs/superpowers/specs/2026-07-13-background-experiment-runs-architecture.md`. +- State: `state/background-runs-prelanding-repair.STATE.md`. +- Reports: `reports/goals/background-runs-prelanding-repair-round-NN.md`. + +## Direction + +Close only the pre-landing blockers established by the PR #2 review: + +1. durable Run Request engine/artifact-version binding; +2. rejection of invalid directly constructed configurations before admission; +3. filesystem Published Artifact immutability after staging mutation; +4. truthful progress for early scientific terminal outcomes; +5. Run ID collision safety; +6. transient SQLite failure classification at the worker boundary; +7. heartbeat/progress persistence by ten ticks or roughly one second; +8. executable scenario-to-pytest mapping in the readiness verifier; +9. full memory/SQLite lifecycle contract parity; +10. `RUNNING.md` alignment: 30 seconds is stretch telemetry, not a release gate. + +## Locked Roles + +- Mutator: primary agent, limited to the approved manifest. +- Evaluator: public-seam pytest contracts plus + `scripts/validate_background_runs_readiness.py`. +- Independent checker: separate read-only agent; it cannot edit tests, verifier, + product code, or success criteria. +- Controller: this Goal, compact STATE, and append-only round reports. + +Public test seams are `ExperimentRuns`, `RunRepository`, +`ArtifactRepository`, `RunWorker`, and the readiness verifier CLI. Tests must +observe caller-visible behavior, adapter contracts, or managed filesystem +effects and must not assert private in-memory fields. + +## Rounds + +1. RED only: add minimal reproductions, full shared lifecycle contracts, exact + scenario-to-pytest mapping, and verifier calibration. Product code and + `RUNNING.md` remain protected. Product mutation unlocks only after the + independent checker accepts the RED evidence. +2. GREEN: make the smallest complete product changes and update `RUNNING.md`. + Do not weaken, delete, skip, or rewrite the Round 1 failures. +3. Closure: run focused and aggregate gates, independent checker, manifest and + scope proof; stop before all Git and release gates. + +## Progress, Stops, and Retention + +- Progress metric: confirmed blocker contracts passing / 9, plus one documentation + contract aligned / 1. +- Failure fingerprint: sorted failing pytest node IDs plus verifier diagnostics. +- No improvement: stop when the same fingerprint repeats across two rounds with + zero newly passing blocker contracts. +- Workload ceiling: no individual simulation may exceed 500 ticks; zero + 5,000-tick executions. +- Stop immediately on scientific artifact/golden drift, sync CLI/default-runner + change, dependency change, or required work outside the approved manifest. +- Human gates: commit, push, merge, default-runner switch, deployment, branch or + worktree cleanup, and 5,000-tick acceptance. +- Retention: keep the reusable verifier and behavioral suites; compact final + STATE, retain all three reports. + +## Deterministic Baseline Reproduction + +The Round 1 protected-file fixture was generated from the exact start ref with: + +```bash +START_REF=d24470505b26bab3cdb0bdfba10276f57aed0862 +for path in RUNNING.md pyproject.toml uv.lock $(git ls-tree -r --name-only "$START_REF" -- antelab | sort); do + digest=$(git show "$START_REF:$path" | /sbin/sha256sum | cut -d' ' -f1) + printf '%s %s\n' "$digest" "$path" +done > tests/manifests/background-runs-prelanding-round-one.sha256 +``` diff --git a/reports/goals/background-runs-prelanding-repair-round-01.md b/reports/goals/background-runs-prelanding-repair-round-01.md new file mode 100644 index 0000000..e6150c9 --- /dev/null +++ b/reports/goals/background-runs-prelanding-repair-round-01.md @@ -0,0 +1,52 @@ +# Background Runs Pre-Landing Repair — Round 01 + +Status: `PASS` + +- Workspace: `/private/tmp/antelab-v1-performance-gate-worktree`. +- Start HEAD: `d24470505b26bab3cdb0bdfba10276f57aed0862`. +- Scope: RED behavioral contracts, scenario mapping, and verifier calibration + only; product implementation and `RUNNING.md` are protected this round. +- Test seams: `ExperimentRuns`, both `RunRepository` adapters, filesystem + `ArtifactRepository`, `RunWorker`, and readiness verifier CLI. +- Tick ceiling: 500 per simulation; 5,000-tick executions: 0. +- Focused RED command: 8 failed, 3 passed, exit 1. The two Run ID collision + failures are one contract exercised against both adapters. +- Aggregate readiness verifier: 8 failed, 38 passed, exit 1. +- Failure fingerprint: + `run_id_collision[memory,sqlite]+request_version_binding+sqlite_transient_io+staging_alias_immutability+invalid_direct_config+time_heartbeat+early_terminal_progress`. +- Positive control: full lifecycle parity passes for memory and SQLite, including + reopen-after-write behavior for the durable adapter. +- Negative control: removing the collected pytest node for the durable request + version scenario makes scenario coverage fail. +- Collection proof: 46 behavioral nodes collected; 30 / 30 locked scenarios have + an exact file/function mapping to at least one collected node. +- Maximum real simulation in the new RED set: 100 requested ticks terminating + at tick 1; no simulation exceeded 500 ticks. +- Product files changed in Round 1: none. +- Independent checker attempt 1: `FAIL`. +- Checker findings repaired without product mutation: + - replaced the obsolete baseline with exact `d244705` hashes for all + `antelab/**/*.py`, `RUNNING.md`, `pyproject.toml`, and `uv.lock`; + - changed version binding RED to admit through the current public API, mutate + the process version, reopen real SQLite, and compare the durable digest; + - changed SQLite transient RED to real lock contention executed through + `RunWorker`, with no private adapter monkeypatch; + - removed the false conflicting-metadata mapping and bound the conflicting + bytes scenario to corrupt-published-byte rejection; + - strengthened invalid-config evidence with ID-allocation and public-get + assertions; + - strengthened heartbeat evidence with observed `renew_lease` timestamps; + - added recovery exhaustion and terminal durability to the same memory/SQLite + lifecycle suite. +- Recalibration: scope, structure, and scenario collection PASS; full adapter + lifecycle 2/2 PASS; missing-node negative control PASS; `git diff --check` + PASS. +- Independent checker attempt 2: `FAIL` only for stale evidence counts in this + report and STATE; all seven substantive findings were closed. Counts are now + corrected to 46 collected behavioral nodes and aggregate 8 failed / 38 passed. +- Independent checker attempt 3: `PASS`. +- Round 2 unlock: only the seven RED product clusters and `RUNNING.md` 30-second + wording may change; Round 1 tests, verifier, mappings, and baseline are locked. +- Baseline reproduction command is recorded in + `goals/background-runs-prelanding-repair.goal.md` and binds every digest to + start ref `d24470505b26bab3cdb0bdfba10276f57aed0862`. diff --git a/reports/goals/background-runs-prelanding-repair-round-02.md b/reports/goals/background-runs-prelanding-repair-round-02.md new file mode 100644 index 0000000..cde2a1e --- /dev/null +++ b/reports/goals/background-runs-prelanding-repair-round-02.md @@ -0,0 +1,34 @@ +# Background Runs Pre-Landing Repair — Round 02 + +Status: `PASS` + +- Unlock: independent Round 1 checker PASS. +- Scope: minimal product GREEN for durable version binding, direct-config + admission, staging alias immutability, early-terminal progress, Run ID + collision safety, SQLite transient classification, and time/tick heartbeat; + align `RUNNING.md` with the accepted 30-second stretch contract. +- Protected: Round 1 tests/verifier/mappings/baseline, synchronous CLI/default + runner, scientific artifact schema/golden, dependencies, Git remote state, + deployment, branches, and worktrees. +- Tick ceiling: 500 per simulation; 5,000-tick executions: 0. +- Product changes: + - `RunRequest` normalizes direct configs and binds engine/artifact versions; + - SQLite schema v2 persists those bindings and rejects digest divergence; + - Run ID collisions are rejected before mutation; + - filesystem publication detaches the staging hard-link and fsyncs both + directory mutations; + - SQLite lock/busy errors become transient I/O at the repository seam; + - RunWorker handles transient claim/state writes, persists by ten ticks or one + second, and forces early-terminal artifact progress before verification; + - artifact ownership now checks engine and artifact schema versions; + - `RUNNING.md` classifies 30 seconds as non-blocking stretch telemetry. +- Focused gate: 11 / 11 PASS. +- Readiness verifier: 46 / 46 PASS. +- Full suite: 601 / 601 PASS. +- Ruff: PASS. Strict mypy: 32 source files PASS. `git diff --check`: PASS. +- First aggregate attempt exposed that read-only published permissions prevented + the existing corruption-detection fixture. The implementation was simplified + to detach the staging alias without changing final-file permissions; both + staging isolation and post-publication corruption detection then passed. +- 5,000-tick executions: 0. +- Next focus: Round 3 independent checker and exact manifest proof. diff --git a/reports/goals/background-runs-prelanding-repair-round-03.md b/reports/goals/background-runs-prelanding-repair-round-03.md new file mode 100644 index 0000000..c822bde --- /dev/null +++ b/reports/goals/background-runs-prelanding-repair-round-03.md @@ -0,0 +1,69 @@ +# Background Runs Pre-Landing Repair — Round 03 + +Status: `PASS` + +- Scope: verification, adversarial review, exact manifest, and stop-state only. +- Product/test mutation: locked unless the checker identifies a concrete defect. +- Initial evidence: focused 11/11; readiness 46/46; full pytest 601/601; Ruff, + strict mypy over 32 source files, and `git diff --check` PASS. +- Test ceiling: at most 500 ticks; real 5,000-tick executions: 0. +- Git/release actions: no commit, push, merge, default-runner switch, deployment, + branch cleanup, or worktree cleanup. +- Independent checker: `PASS` on attempt 2 after three bounded corrections. +- Independent checker attempt 1: `FAIL` with three reproduced findings: + writable staging alias after a publish crash, incomplete parameterized adapter + mapping, and missing deterministic fixture reproduction command. +- Corrections: + - publication copies staged bytes to a separate fsynced publishing inode; + recovery removes and fsyncs any same-inode staging/publishing aliases; + - scenario mappings enumerate exact memory/SQLite and memory/filesystem nodes, + with a negative calibration that removes only the SQLite node; + - the Goal records the exact start-ref baseline generation command. +- Corrected focused gate: 5 / 5 PASS. +- Corrected readiness verifier: 47 / 47 PASS. +- Corrected full suite: 603 / 603 PASS. +- Ruff: PASS. Strict mypy: 32 source files PASS. `git diff --check`: PASS. +- Independent checker attempt 2: `PASS`. +- Protected-surface proof: no diff in `antelab/cli.py`, dependency files, + default-runner behavior, `antelab/artifacts/schema.py`, golden artifacts, or + synchronous runner behavior. +- 5,000-tick executions: 0. + +## Exact 22-File Manifest + +Product and public documentation: + +- `RUNNING.md` +- `antelab/runs/artifacts.py` +- `antelab/runs/model.py` +- `antelab/runs/ownership.py` +- `antelab/runs/repository.py` +- `antelab/runs/sqlite_repository.py` +- `antelab/runs/worker.py` + +Verifier and deterministic fixtures: + +- `scripts/validate_background_runs_readiness.py` +- `tests/manifests/background-run-scenarios.txt` +- `tests/manifests/background-run-scenario-tests.txt` +- `tests/manifests/background-runs-prelanding-round-one.sha256` + +Behavioral contracts: + +- `tests/runs/test_adapter_contracts.py` +- `tests/runs/test_durable_adapters.py` +- `tests/runs/test_experiment_runs.py` +- `tests/runs/test_inprocess_engine.py` +- `tests/runs/test_run_worker.py` +- `tests/test_background_runs_verifier.py` + +Goal control and evidence: + +- `goals/background-runs-prelanding-repair.goal.md` +- `state/background-runs-prelanding-repair.STATE.md` +- `reports/goals/background-runs-prelanding-repair-round-01.md` +- `reports/goals/background-runs-prelanding-repair-round-02.md` +- `reports/goals/background-runs-prelanding-repair-round-03.md` + +Stop: objective achieved. No commit, push, merge, 5,000-tick execution, +default-runner switch, deployment, branch cleanup, or worktree cleanup. diff --git a/scripts/validate_background_runs_readiness.py b/scripts/validate_background_runs_readiness.py index 7eee269..50ed337 100644 --- a/scripts/validate_background_runs_readiness.py +++ b/scripts/validate_background_runs_readiness.py @@ -34,6 +34,13 @@ progress_persistence_is_bounded transient_repository_io_is_recoverable corrupt_artifact_is_never_returned +run_request_version_binding_survives_restart +invalid_direct_config_is_rejected_before_admission +published_artifact_isolated_from_staging_aliases +early_terminal_progress_matches_artifact +run_id_collision_preserves_existing_run +sqlite_operational_error_is_transient +progress_heartbeat_uses_tick_or_time_cadence """.splitlines() if line.strip() } @@ -48,7 +55,8 @@ "antelab/runs/recovery.py", } -ROUND_ONE_BASELINE = "tests/manifests/background-runs-round-one.sha256" +ROUND_ONE_BASELINE = "tests/manifests/background-runs-prelanding-round-one.sha256" +SCENARIO_TEST_MAP = "tests/manifests/background-run-scenario-tests.txt" BEHAVIORAL_TESTS = ( @@ -71,12 +79,95 @@ def validate_structure(root: Path) -> tuple[str, ...]: } if manifest != EXPECTED_SCENARIOS: errors.append("scenario manifest differs from the locked expected set") + mapping_path = root / SCENARIO_TEST_MAP + if not mapping_path.is_file(): + errors.append("scenario-to-test mapping is missing") + else: + mapping, mapping_errors = _load_scenario_test_mapping(mapping_path) + errors.extend(mapping_errors) + if set(mapping) != EXPECTED_SCENARIOS: + errors.append("scenario-to-test mapping differs from the locked set") missing_files = sorted(path for path in REQUIRED_FILES if not (root / path).is_file()) if missing_files: errors.append(f"required files missing: {missing_files}") return tuple(errors) +def collect_behavioral_nodes(root: Path) -> tuple[set[str], str | None]: + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--collect-only", + "-q", + *BEHAVIORAL_TESTS, + ], + cwd=root, + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + diagnostic = (result.stdout + result.stderr).strip() + return set(), f"behavioral pytest collection failed: {diagnostic[-1000:]}" + nodes = { + line.strip() + for line in result.stdout.splitlines() + if line.strip().startswith("tests/") and "::" in line + } + return nodes, None + + +def validate_scenario_coverage( + root: Path, collected_nodes: set[str] | None = None +) -> tuple[str, ...]: + mapping_path = root / SCENARIO_TEST_MAP + if not mapping_path.is_file(): + return ("scenario-to-test mapping is missing",) + mapping, errors = _load_scenario_test_mapping(mapping_path) + if collected_nodes is None: + collected_nodes, collection_error = collect_behavioral_nodes(root) + if collection_error is not None: + return (*errors, collection_error) + missing = sorted( + f"{scenario}:{expected}" + for scenario, expected_nodes in mapping.items() + for expected in expected_nodes + if not any( + node == expected or node.startswith(f"{expected}[") + for node in collected_nodes + ) + ) + if missing: + errors.append(f"scenarios without collected pytest nodes: {missing}") + return tuple(errors) + + +def _load_scenario_test_mapping( + path: Path, +) -> tuple[dict[str, tuple[str, ...]], list[str]]: + mapping: dict[str, tuple[str, ...]] = {} + errors: list[str] = [] + for line_number, raw_line in enumerate( + path.read_text(encoding="utf-8").splitlines(), start=1 + ): + line = raw_line.strip() + if not line: + continue + parts = line.split() + nodes = tuple(parts[1].split(",")) if len(parts) == 2 else () + if len(parts) != 2 or not nodes or any("::test_" not in node for node in nodes): + errors.append(f"invalid scenario mapping at line {line_number}") + continue + scenario = parts[0] + if scenario in mapping: + errors.append(f"duplicate scenario mapping: {scenario}") + continue + mapping[scenario] = nodes + return mapping, errors + + def validate_round_one_scope(root: Path) -> tuple[str, ...]: errors: list[str] = [] baseline_path = root / ROUND_ONE_BASELINE @@ -119,6 +210,8 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) root = args.root.resolve() errors = validate_structure(root) + if not errors: + errors += validate_scenario_coverage(root) if args.round_one_scope: errors += validate_round_one_scope(root) if errors: diff --git a/state/background-runs-prelanding-repair.STATE.md b/state/background-runs-prelanding-repair.STATE.md new file mode 100644 index 0000000..6deccf1 --- /dev/null +++ b/state/background-runs-prelanding-repair.STATE.md @@ -0,0 +1,65 @@ +# State: Background Runs Pre-Landing Repair + +Schema version: 1 + +- Status: `succeeded`. +- Round: 3 / 3. +- Authority: closure verification and independent checker only. +- Worktree: `/private/tmp/antelab-v1-performance-gate-worktree`. +- Branch: `feature/v1-performance-gate`. +- Start HEAD: `d24470505b26bab3cdb0bdfba10276f57aed0862`. +- Start dirty state: clean. +- Product mutation: unlocked by independent checker Round 1 PASS. +- Test workload: at most 500 ticks; 5,000-tick executions forbidden. +- Progress: 9 / 9 blocker contracts closed; documentation alignment complete. +- Current evidence: final local gates and independent checker PASS; remote CI + remains bound to pre-repair commit `d244705` and is not evidence for this WIP. +- Failure fingerprint: + `run_id_collision[memory,sqlite]+request_version_binding+sqlite_transient_io+staging_alias_immutability+invalid_direct_config+time_heartbeat+early_terminal_progress`. +- No-improvement count: 0 / 2. +- Protected: synchronous CLI/default runner, scientific artifact/golden, + dependency lock, Git remote state, deployment, branches, and worktrees. +- Round 1 focused RED: 8 failed, 3 passed; two failures are the same Run ID + collision contract across memory and SQLite. +- Readiness verifier RED: 8 failed, 38 passed; exit 1. +- Full lifecycle parity: memory and SQLite PASS with reconstruction after durable + mutations. +- Verifier negative calibration: removing one collected pytest node is rejected. +- Scenario mapping: 30 / 30 scenario IDs map to collected pytest nodes. +- Independent checker attempt 1: `FAIL`; rejected stale scope baseline, one + API-shape fake RED, a private SQLite fault seam, two semantic mapping gaps, + weak zero-admission/heartbeat evidence, and incomplete recovery-exhaustion + adapter parity. +- Checker repairs: start-ref scope baseline now passes; version RED uses a + post-admission version change and real SQLite reopen; transient SQLite RED uses + real database lock contention through RunWorker; mappings are semantic; + invalid admission proves no ID allocation/public retrieval; heartbeat records + renew and progress; recovery exhaustion runs through both adapters. +- Independent checker attempt 2: `FAIL` only because this STATE and the Round 1 + report retained pre-repair test counts; no substantive contract finding + remained. Evidence corrected to 46 collected behavioral nodes and aggregate + 8 failed / 38 passed. +- Independent checker attempt 3: `PASS`. +- Round 2 focused gate: 11 / 11 PASS. +- Readiness verifier: 47 / 47 PASS after final checker corrections. +- Static gates: Ruff PASS; strict mypy PASS across 32 source files; + `git diff --check` PASS. +- Full pytest: 603 / 603 PASS after final checker corrections. +- `RUNNING.md`: 30 seconds is non-blocking stretch telemetry; 120 seconds and + 512 MiB remain the separately gated 5,000-tick closure budget. +- Maximum authorized test workload: 500 ticks; real 5,000-tick executions: 0. +- Final checker attempt 1: `FAIL`; required crash-window alias recovery, + exact parameterized adapter-node coverage, and a deterministic baseline + reproduction command. +- Checker corrections: publication now copies to a durable publishing inode and + removes same-inode staging/publishing aliases on recovery; parameterized + mappings enumerate every required adapter; the exact start-ref baseline + command is recorded in the Goal. +- Independent final checker attempt 2: `PASS`. +- Final gates: focused correction 5 / 5; readiness 47 / 47; full pytest + 603 / 603; Ruff PASS; strict mypy 32 source files PASS; diff check PASS. +- Final manifest: 22 files, recorded in Round 03. +- Stop reason: implementation and verification objective achieved; waiting at + explicit commit/push/merge/default-runner/deploy/cleanup human gates. +- Next focus: human review of the exact manifest; no Git mutation is authorized. +- Updated: 2026-07-14T04:02:00Z. diff --git a/tests/manifests/background-run-scenario-tests.txt b/tests/manifests/background-run-scenario-tests.txt new file mode 100644 index 0000000..06fc656 --- /dev/null +++ b/tests/manifests/background-run-scenario-tests.txt @@ -0,0 +1,30 @@ +submit_same_key_same_request_deduplicates tests/runs/test_experiment_runs.py::test_submit_is_idempotent_and_conflicting_reuse_is_rejected +submit_same_key_different_request_conflicts tests/runs/test_experiment_runs.py::test_submit_is_idempotent_and_conflicting_reuse_is_rejected +concurrent_equal_submissions_create_one_run tests/runs/test_adapter_contracts.py::test_concurrent_equal_submissions_create_one_run[memory],tests/runs/test_adapter_contracts.py::test_concurrent_equal_submissions_create_one_run[sqlite] +cancel_queued_is_terminal_and_idempotent tests/runs/test_experiment_runs.py::test_cancel_queued_run_is_terminal_and_idempotent +claim_has_single_fenced_owner tests/runs/test_run_repository.py::test_claim_is_exclusive_and_stale_fence_cannot_progress +stale_fence_cannot_progress_or_complete tests/runs/test_run_repository.py::test_expired_fence_cannot_verify_or_complete +running_cancel_beats_completion tests/runs/test_run_repository.py::test_cancel_and_completion_have_one_compare_and_swap_winner +completion_beats_late_cancel tests/runs/test_run_repository.py::test_cancel_and_completion_have_one_compare_and_swap_winner +expired_attempt_replays_from_tick_zero tests/runs/test_run_repository.py::test_expired_attempt_requeues_from_zero_without_losing_max_tick +recovery_exhaustion_fails tests/runs/test_run_repository.py::test_repeated_worker_loss_exhausts_recovery_budget +memory_sqlite_repository_contract_parity tests/runs/test_adapter_contracts.py::test_run_repository_full_lifecycle_contract[memory],tests/runs/test_adapter_contracts.py::test_run_repository_full_lifecycle_contract[sqlite] +memory_filesystem_artifact_contract_parity tests/runs/test_adapter_contracts.py::test_artifact_repository_shared_contract[memory],tests/runs/test_adapter_contracts.py::test_artifact_repository_shared_contract[filesystem] +staged_artifact_is_not_visible tests/runs/test_durable_adapters.py::test_filesystem_artifact_is_invisible_until_immutable_publish +immutable_publish_deduplicates_same_bytes tests/runs/test_adapter_contracts.py::test_artifact_repository_shared_contract[memory],tests/runs/test_adapter_contracts.py::test_artifact_repository_shared_contract[filesystem] +immutable_publish_rejects_conflicting_bytes tests/runs/test_durable_adapters.py::test_corrupt_published_artifact_is_never_returned +publish_then_crash_reconciles_once tests/runs/test_durable_adapters.py::test_publish_then_restart_reconciles_completed_run_once +short_run_progress_is_truthful tests/runs/test_run_worker.py::test_worker_publishes_one_completed_artifact +short_run_cancel_publishes_no_artifact tests/runs/test_run_worker.py::test_running_cancel_publishes_no_artifact +short_run_artifact_matches_reference_bytes tests/runs/test_inprocess_engine.py::test_background_short_run_matches_synchronous_reference_artifact +real_inprocess_run_is_observable_and_cancellable tests/runs/test_inprocess_engine.py::test_real_inprocess_run_remains_observable_and_cancellable +progress_persistence_is_bounded tests/runs/test_run_worker.py::test_progress_persistence_is_bounded_to_cadence_and_terminal_tick +transient_repository_io_is_recoverable tests/runs/test_run_worker.py::test_transient_repository_io_is_not_permanent_engine_failure +corrupt_artifact_is_never_returned tests/runs/test_durable_adapters.py::test_corrupt_published_artifact_is_never_returned +run_request_version_binding_survives_restart tests/runs/test_durable_adapters.py::test_run_request_version_binding_survives_adapter_reconstruction +invalid_direct_config_is_rejected_before_admission tests/runs/test_experiment_runs.py::test_invalid_direct_config_is_rejected_before_run_admission +published_artifact_isolated_from_staging_aliases tests/runs/test_durable_adapters.py::test_publish_recovery_detaches_crash_window_staging_alias +early_terminal_progress_matches_artifact tests/runs/test_inprocess_engine.py::test_early_terminal_progress_matches_published_artifact +run_id_collision_preserves_existing_run tests/runs/test_adapter_contracts.py::test_run_id_collision_preserves_existing_run[memory],tests/runs/test_adapter_contracts.py::test_run_id_collision_preserves_existing_run[sqlite] +sqlite_operational_error_is_transient tests/runs/test_durable_adapters.py::test_sqlite_lock_contention_is_recoverable_at_worker_boundary +progress_heartbeat_uses_tick_or_time_cadence tests/runs/test_run_worker.py::test_progress_heartbeat_uses_tick_or_time_cadence diff --git a/tests/manifests/background-run-scenarios.txt b/tests/manifests/background-run-scenarios.txt index 62ba551..380d34a 100644 --- a/tests/manifests/background-run-scenarios.txt +++ b/tests/manifests/background-run-scenarios.txt @@ -21,3 +21,10 @@ real_inprocess_run_is_observable_and_cancellable progress_persistence_is_bounded transient_repository_io_is_recoverable corrupt_artifact_is_never_returned +run_request_version_binding_survives_restart +invalid_direct_config_is_rejected_before_admission +published_artifact_isolated_from_staging_aliases +early_terminal_progress_matches_artifact +run_id_collision_preserves_existing_run +sqlite_operational_error_is_transient +progress_heartbeat_uses_tick_or_time_cadence diff --git a/tests/manifests/background-runs-prelanding-round-one.sha256 b/tests/manifests/background-runs-prelanding-round-one.sha256 new file mode 100644 index 0000000..a4998fc --- /dev/null +++ b/tests/manifests/background-runs-prelanding-round-one.sha256 @@ -0,0 +1,32 @@ +59f03a890554bb9f8a4215a93eb3b0ee8eb9bc7cf505c4fc5c03ca4a7749d1b3 RUNNING.md +59b4a06a60d25eb7717006749fd1f9f4dfff1cb504ced42c0b7ab56675c2e466 pyproject.toml +8f6cf989e8f3e7e4e8879ce80926064824719093872c36bf6ee2e239d00d7bc9 uv.lock +4059ae61c26515a4a9c8f69f8d044141670c8e77a851a6b019a549a43c37fa59 antelab/__init__.py +5d1722fa9b82a6bace7ea2d44570bd25db688b0fab539a5e0bb35fe1468b35f9 antelab/artifacts/__init__.py +dc0ca5b31eb30755289f0a3494b4a7022bf44d70dfb054c31f2c62683092b1b1 antelab/artifacts/schema.py +4b57da98ba49841d2a66f4667254ee2a685416f4c86efc56b908dfcd429cded1 antelab/artifacts/writer.py +d834f20c560e69ebd0029bf9f9966bc4fc5f6a3d68982ca7897a2543fc45164f antelab/cli.py +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 antelab/core/__init__.py +0e95c6fd40ae468766907894c20eb8ed5cf2a5792991db1f7b3fb1cc08fd071f antelab/core/brain.py +63e1685770799227bd15e573d48fc9061cabd07a5b760dabc44daea5186c62f8 antelab/core/config.py +e52c94fe52291c232cb9bb7da5bac3fc135c3e417b2140baef775296afaca663 antelab/core/environment.py +f4eeda895ddb746427d5b4ea512fe47adca3b55840500cdebcc8f1d001b5dedc antelab/core/evolution.py +80cc482df0329c98a992fc0368b4e444190b069e39c36e5a89b69d1ed3dc717d antelab/core/fixed.py +d45bfac76cb7abf603363f904d83f99371560a7441cd03e5e176a665390a8f1b antelab/core/genome.py +490e3ec3042bcded932a832c6a4e3be1166f7ffab7fe4042194b2cee6405cc1f antelab/core/organism.py +d1367496fb792de2e1edacd8797bf164b66d40fc52b5137c4a69bf5c5858424e antelab/core/physics.py +6543b69bdd97b276d50d6e0dda0ac9df2ddcaeab24a433943520089a5756fea4 antelab/core/rng.py +48c8cb7252608a036b1d053575fb268f610baec099698797fac7518d7909d31b antelab/core/simulation.py +dcb5cfabd35514716a33fca1f99553387f8fa13eed7a0926e1a6a384050bca2f antelab/core/spatial.py +adadeaff04a392ce1e9e750189f645a0a1be52150bb981d91e76a626066801d7 antelab/experiments/__init__.py +63c2cec5988ac21eab6de035fa349612fac77b367fd4b247ff663f085af514c0 antelab/experiments/runner.py +afaf36c47c1c18edb93dcc1196127b00e7d54b664e95d0901f8e715b8e52ae4c antelab/runs/__init__.py +f6767005094d9b9f1a3704f467b13aed3e8e480f544d6709fd5ad54ac727ecc4 antelab/runs/artifacts.py +86bc36304ca28ca8129a10d75115410f9239917cc2c69bbe087b3e5edfb07011 antelab/runs/errors.py +34d13e245f5a5d20ce7044a7e163ddd7526cd2df500f20de71fbff59ba53deac antelab/runs/manager.py +79dd51207502834381d92df747f1c61763ea5df0240d791634e6cd2eaee99ac4 antelab/runs/model.py +3e6bcecb226d6ee3f05600c5fcc745b9ca86acaf07ed808582ab5c14627cf2ad antelab/runs/ownership.py +a13816116b3dfba0054c5376629fa40270c08a547116dd2c664bf43ce985bdcc antelab/runs/recovery.py +ed1513e573f47a42bd9a0b02ff9a9a80c3986b01acc31046127c55fb9811ac0e antelab/runs/repository.py +92a64a6c1070ac48fbae252537e9b94f9005f199b6f384748eb12a340cef190c antelab/runs/sqlite_repository.py +9d4be628126d0fd15eddff7aebcbca3f3e7392c313ee69b7b4a0e0de1aa1e6dc antelab/runs/worker.py diff --git a/tests/runs/test_adapter_contracts.py b/tests/runs/test_adapter_contracts.py index 9844e6c..d28a6e6 100644 --- a/tests/runs/test_adapter_contracts.py +++ b/tests/runs/test_adapter_contracts.py @@ -1,3 +1,4 @@ +from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor from dataclasses import replace from datetime import UTC, datetime, timedelta @@ -103,6 +104,130 @@ def assert_artifact_repository_contract( assert repository.publish(reference) == reference +def assert_full_run_repository_lifecycle_contract( + open_repository: Callable[[], RunRepository], +) -> None: + repository = open_repository() + request = _request() + + created = repository.create_idempotent( + request, IdempotencyKey("full-success"), lambda: RunId("full-success") + ) + repository = open_repository() + lease = repository.claim_next( + now=NOW, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "success-fence", + ) + assert lease is not None and lease.run_id == created.run_id + progressed = repository.record_progress(lease, tick=1, now=NOW) + lease = replace(lease, revision=progressed.revision) + repository = open_repository() + verifying = repository.begin_verifying(lease, now=NOW) + lease = replace(lease, revision=verifying.revision) + repository = open_repository() + reference = ArtifactRef( + artifact_id=f"sha256/{'a' * 64}", + sha256="a" * 64, + byte_size=1, + schema_version=1, + ) + publishing = repository.begin_publishing( + lease, artifact_id=reference.artifact_id, now=NOW + ) + lease = replace(lease, revision=publishing.revision) + repository = open_repository() + completed = repository.complete(lease, artifact=reference, now=NOW) + assert completed.state == "completed" + assert open_repository().get(created.run_id) == completed + + repository = open_repository() + failed_run = repository.create_idempotent( + request, IdempotencyKey("full-failure"), lambda: RunId("full-failure") + ) + lease = repository.claim_next( + now=NOW, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "failure-fence", + ) + assert lease is not None and lease.run_id == failed_run.run_id + failed = repository.fail( + lease, + code="contract_failure", + message="contract failure", + retryable=False, + now=NOW, + ) + assert open_repository().get(failed_run.run_id) == failed + + repository = open_repository() + queued_cancel = repository.create_idempotent( + request, + IdempotencyKey("full-queued-cancel"), + lambda: RunId("full-queued-cancel"), + ) + cancelled = repository.cancel(queued_cancel.run_id) + assert cancelled.state == "cancelled" + assert open_repository().get(queued_cancel.run_id) == cancelled + + repository = open_repository() + running_cancel = repository.create_idempotent( + request, + IdempotencyKey("full-running-cancel"), + lambda: RunId("full-running-cancel"), + ) + lease = repository.claim_next( + now=NOW, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "cancel-fence", + ) + assert lease is not None and lease.run_id == running_cancel.run_id + cancelling = repository.cancel(running_cancel.run_id) + lease = replace(lease, revision=cancelling.revision) + repository = open_repository() + acknowledged = repository.acknowledge_cancel(lease, now=NOW) + assert acknowledged.state == "cancelled" + assert open_repository().get(running_cancel.run_id) == acknowledged + + repository = open_repository() + recovering = repository.create_idempotent( + request, + IdempotencyKey("full-recovery"), + lambda: RunId("full-recovery"), + ) + lease = repository.claim_next( + now=NOW, + lease_duration=timedelta(seconds=10), + token_factory=lambda: "recovery-fence", + ) + assert lease is not None and lease.run_id == recovering.run_id + repository.record_progress(lease, tick=1, now=NOW) + repository = open_repository() + replayed = repository.recover_expired(now=NOW + timedelta(seconds=11)) + assert len(replayed) == 1 + assert replayed[0].run_id == recovering.run_id + assert replayed[0].state == "queued" + assert replayed[0].progress.attempt_tick == 0 + assert replayed[0].progress.max_tick_seen == 1 + assert open_repository().get(recovering.run_id) == replayed[0] + now = NOW + timedelta(seconds=11) + for attempt in (2, 3): + repository = open_repository() + lease = repository.claim_next( + now=now, + lease_duration=timedelta(seconds=10), + token_factory=lambda attempt=attempt: f"recovery-fence-{attempt}", + ) + assert lease is not None and lease.attempt == attempt + now += timedelta(seconds=11) + recovered = repository.recover_expired(now=now) + assert len(recovered) == 1 + assert recovered[0].state == "failed" + assert recovered[0].failure is not None + assert recovered[0].failure.code == "worker_recovery_exhausted" + assert open_repository().get(recovering.run_id) == recovered[0] + + @pytest.mark.parametrize("adapter", ["memory", "sqlite"]) def test_run_repository_shared_contract(adapter: str, tmp_path: Path) -> None: repository: RunRepository @@ -113,6 +238,52 @@ def test_run_repository_shared_contract(adapter: str, tmp_path: Path) -> None: assert_run_repository_contract(repository) +@pytest.mark.parametrize("adapter", ["memory", "sqlite"]) +def test_run_repository_full_lifecycle_contract( + adapter: str, tmp_path: Path +) -> None: + memory_repository = InMemoryRunRepository() if adapter == "memory" else None + database = tmp_path / "full-lifecycle.sqlite3" + + def open_repository() -> RunRepository: + if memory_repository is not None: + return memory_repository + return SqliteRunRepository(database) + + assert_full_run_repository_lifecycle_contract(open_repository) + + +@pytest.mark.parametrize("adapter", ["memory", "sqlite"]) +def test_run_id_collision_preserves_existing_run( + adapter: str, tmp_path: Path +) -> None: + repository: RunRepository + if adapter == "memory": + repository = InMemoryRunRepository() + else: + repository = SqliteRunRepository(tmp_path / "run-id-collision.sqlite3") + first_request = _request() + first = repository.create_idempotent( + first_request, + IdempotencyKey("collision-first"), + lambda: RunId("same-run-id"), + ) + + with pytest.raises(RunStateConflictError, match="run id"): + repository.create_idempotent( + RunRequest(replace(first_request.config, ticks=4)), + IdempotencyKey("collision-second"), + lambda: RunId("same-run-id"), + ) + + assert repository.get(first.run_id) == first + assert repository.create_idempotent( + first_request, + IdempotencyKey("collision-first"), + lambda: RunId("unused"), + ) == first + + @pytest.mark.parametrize("adapter", ["memory", "sqlite"]) def test_concurrent_equal_submissions_create_one_run( adapter: str, tmp_path: Path diff --git a/tests/runs/test_durable_adapters.py b/tests/runs/test_durable_adapters.py index b2a84a4..ddc0638 100644 --- a/tests/runs/test_durable_adapters.py +++ b/tests/runs/test_durable_adapters.py @@ -1,19 +1,27 @@ +import sqlite3 +from collections.abc import Callable from dataclasses import replace from datetime import timedelta from pathlib import Path import pytest +import antelab.runs.model as run_model from antelab.artifacts import RunArtifact from antelab.core.config import load_config from antelab.core.simulation import Simulation from antelab.experiments.runner import _metric -from antelab.runs import RunRequest +from antelab.runs import ( + ExperimentRuns, + InMemoryArtifactRepository, + RunRequest, +) from antelab.runs.artifacts import FilesystemArtifactRepository from antelab.runs.errors import ArtifactIntegrityError from antelab.runs.model import IdempotencyKey, RunId from antelab.runs.recovery import reconcile_publications from antelab.runs.sqlite_repository import SqliteRunRepository +from antelab.runs.worker import RunWorker def _request() -> RunRequest: @@ -46,6 +54,63 @@ def test_sqlite_repository_survives_adapter_reconstruction(tmp_path: Path) -> No assert SqliteRunRepository(database).get(created.run_id).state == "cancelled" +def test_run_request_version_binding_survives_adapter_reconstruction( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + database = tmp_path / "version-bound-runs.sqlite3" + request = _request() + first = SqliteRunRepository(database) + created = first.create_idempotent( + request, + IdempotencyKey("version-bound"), + lambda: RunId("version-bound-run"), + ) + + monkeypatch.setattr(run_model, "__version__", "99.0.0-test-upgrade") + restored = SqliteRunRepository(database).request(created.run_id) + + assert restored == request + assert restored.digest() == created.request_digest + + +def test_sqlite_lock_contention_is_recoverable_at_worker_boundary( + tmp_path: Path, +) -> None: + database = tmp_path / "locked.sqlite3" + repository = SqliteRunRepository(database) + artifacts = InMemoryArtifactRepository() + runs = ExperimentRuns(repository, artifacts, id_generator=lambda: "locked-run") + submitted = runs.submit(_request(), idempotency_key="locked") + artifact = _artifact() + + class LockingEngine: + def execute( + self, request: RunRequest, progress: Callable[[int], bool] + ) -> RunArtifact: + locker = sqlite3.connect(database, timeout=0) + locker.execute("BEGIN IMMEDIATE") + try: + progress(request.config.ticks) + finally: + locker.rollback() + locker.close() + return artifact + + worker = RunWorker( + repository, + artifacts, + LockingEngine(), + now=lambda: submitted.progress.updated_at, + token_factory=lambda: "locked-fence", + lease_duration=timedelta(seconds=30), + ) + + assert worker.run_once() is True + waiting = runs.get(submitted.run_id) + assert waiting.state == "running" + assert waiting.failure is None + + def test_filesystem_artifact_is_invisible_until_immutable_publish( tmp_path: Path, ) -> None: @@ -64,6 +129,40 @@ def test_filesystem_artifact_is_invisible_until_immutable_publish( assert reopened.load(reference).to_dict() == artifact.to_dict() +def test_published_artifact_is_isolated_from_staging_aliases(tmp_path: Path) -> None: + root = tmp_path / "artifacts" + repository = FilesystemArtifactRepository(root) + artifact = _artifact() + reference = repository.stage(artifact) + staged = next((root / ".staging").glob("*.tmp")) + + repository.publish(reference) + if staged.exists(): + staged.write_bytes(b"mutated staging bytes") + + assert repository.load(reference).to_dict() == artifact.to_dict() + + +def test_publish_recovery_detaches_crash_window_staging_alias( + tmp_path: Path, +) -> None: + root = tmp_path / "artifacts" + repository = FilesystemArtifactRepository(root) + artifact = _artifact() + reference = repository.stage(artifact) + staged = next((root / ".staging").glob("*.tmp")) + final = root / reference.artifact_id + final.parent.mkdir(parents=True, exist_ok=True) + final.hardlink_to(staged) + assert final.samefile(staged) + + reopened = FilesystemArtifactRepository(root) + assert reopened.publish(reference) == reference + + assert not staged.exists() + assert reopened.load(reference).to_dict() == artifact.to_dict() + + def test_publish_then_restart_reconciles_completed_run_once(tmp_path: Path) -> None: database = tmp_path / "runs.sqlite3" repository = SqliteRunRepository(database) diff --git a/tests/runs/test_experiment_runs.py b/tests/runs/test_experiment_runs.py index 5678118..900939c 100644 --- a/tests/runs/test_experiment_runs.py +++ b/tests/runs/test_experiment_runs.py @@ -3,12 +3,14 @@ import pytest -from antelab.core.config import load_config +from antelab.core.config import ConfigError, load_config from antelab.runs import ( ExperimentRuns, IdempotencyConflictError, InMemoryArtifactRepository, InMemoryRunRepository, + RunId, + RunNotFoundError, RunRequest, ) @@ -52,3 +54,33 @@ def test_cancel_queued_run_is_terminal_and_idempotent() -> None: assert cancelled.artifact is None assert runs.cancel(submitted.run_id) == cancelled assert runs.get(submitted.run_id) == cancelled + + +def test_invalid_direct_config_is_rejected_before_run_admission() -> None: + config = load_config(Path("experiments/foraging-genesis.yaml")) + invalid = replace( + config, + life=replace(config.life, initial_population=-1), + ) + repository = InMemoryRunRepository() + generated: list[str] = [] + + def generate_id() -> str: + generated.append("invalid-run") + return "invalid-run" + + runs = ExperimentRuns( + repository, + InMemoryArtifactRepository(), + id_generator=generate_id, + ) + + with pytest.raises(ConfigError): + runs.submit( + RunRequest(invalid), + idempotency_key="invalid-direct-config", + ) + + assert generated == [] + with pytest.raises(RunNotFoundError): + runs.get(RunId("invalid-run")) diff --git a/tests/runs/test_inprocess_engine.py b/tests/runs/test_inprocess_engine.py index 79557c8..845042e 100644 --- a/tests/runs/test_inprocess_engine.py +++ b/tests/runs/test_inprocess_engine.py @@ -87,3 +87,34 @@ def test_real_inprocess_run_remains_observable_and_cancellable() -> None: assert observed_running assert not thread.is_alive() assert runs.get(submitted.run_id).state == "cancelled" + + +def test_early_terminal_progress_matches_published_artifact() -> None: + base = load_config(Path("experiments/foraging-genesis.yaml")) + config = replace( + base, + ticks=100, + life=replace(base.life, initial_population=1, max_age=1), + ) + repository = InMemoryRunRepository() + artifacts = InMemoryArtifactRepository() + runs = ExperimentRuns(repository, artifacts, id_generator=lambda: "early-terminal") + submitted = runs.submit( + RunRequest(config), idempotency_key="early-terminal-progress" + ) + worker = RunWorker( + repository, + artifacts, + InProcessSimulationEngine(), + now=lambda: datetime(2026, 7, 14, tzinfo=UTC), + token_factory=lambda: "fence-early", + lease_duration=timedelta(seconds=10), + ) + + assert worker.run_once() is True + + completed = runs.get(submitted.run_id) + artifact = runs.artifact(submitted.run_id) + assert artifact.ticks_completed < artifact.ticks_requested + assert completed.progress.attempt_tick == artifact.ticks_completed + assert completed.progress.max_tick_seen == artifact.ticks_completed diff --git a/tests/runs/test_run_worker.py b/tests/runs/test_run_worker.py index 7fc2722..6745f5e 100644 --- a/tests/runs/test_run_worker.py +++ b/tests/runs/test_run_worker.py @@ -243,6 +243,53 @@ def record_progress(self, lease, *, tick, now): # type: ignore[no-untyped-def] assert repository.persisted_ticks == [10, 20, 25] +def test_progress_heartbeat_uses_tick_or_time_cadence() -> None: + class RecordingRepository(InMemoryRunRepository): + def __init__(self) -> None: + super().__init__() + self.persisted_ticks: list[int] = [] + self.renewed_at: list[datetime] = [] + + def renew_lease( # type: ignore[no-untyped-def] + self, lease, *, now, lease_duration + ): + self.renewed_at.append(now) + return super().renew_lease( + lease, now=now, lease_duration=lease_duration + ) + + def record_progress(self, lease, *, tick, now): # type: ignore[no-untyped-def] + self.persisted_ticks.append(tick) + return super().record_progress(lease, tick=tick, now=now) + + clock = [NOW] + + def advance_clock(_: int) -> None: + clock[0] += timedelta(milliseconds=600) + + repository = RecordingRepository() + artifacts = InMemoryArtifactRepository() + runs = ExperimentRuns(repository, artifacts, id_generator=lambda: "run-1") + worker = RunWorker( + repository, + artifacts, + DeterministicFakeEngine( + _artifact(3), ticks=(1, 2, 3), on_tick=advance_clock + ), + now=lambda: clock[0], + token_factory=lambda: "fence-1", + lease_duration=timedelta(seconds=10), + ) + runs.submit(_request(3), idempotency_key="time-cadence") + + assert worker.run_once() is True + assert repository.persisted_ticks == [2, 3] + assert repository.renewed_at == [ + NOW + timedelta(milliseconds=1200), + NOW + timedelta(milliseconds=1800), + ] + + def test_transient_repository_io_is_not_permanent_engine_failure() -> None: class TransientProgressRepository(InMemoryRunRepository): def record_progress(self, lease, *, tick, now): # type: ignore[no-untyped-def] diff --git a/tests/test_background_runs_verifier.py b/tests/test_background_runs_verifier.py index ff953da..7edd998 100644 --- a/tests/test_background_runs_verifier.py +++ b/tests/test_background_runs_verifier.py @@ -2,17 +2,61 @@ from pathlib import Path from scripts.validate_background_runs_readiness import ( + collect_behavioral_nodes, run_behavioral_contract, validate_round_one_scope, + validate_scenario_coverage, validate_structure, ) def test_background_runs_verifier_accepts_current_known_good() -> None: assert validate_structure(Path.cwd()) == () + assert validate_scenario_coverage(Path.cwd()) == () assert run_behavioral_contract(Path.cwd()) == 0 +def test_background_runs_verifier_rejects_uncollected_scenario_node() -> None: + root = Path.cwd() + collected, collection_error = collect_behavioral_nodes(root) + assert collection_error is None + removed = { + node + for node in collected + if node.startswith( + "tests/runs/test_durable_adapters.py::" + "test_run_request_version_binding_survives_adapter_reconstruction" + ) + } + assert removed + + errors = validate_scenario_coverage(root, collected - removed) + + assert any( + "run_request_version_binding_survives_restart" in error + for error in errors + ) + + +def test_background_runs_verifier_requires_every_parameterized_adapter() -> None: + root = Path.cwd() + collected, collection_error = collect_behavioral_nodes(root) + assert collection_error is None + sqlite_node = ( + "tests/runs/test_adapter_contracts.py::" + "test_run_repository_full_lifecycle_contract[sqlite]" + ) + assert sqlite_node in collected + + errors = validate_scenario_coverage(root, collected - {sqlite_node}) + + assert any( + "memory_sqlite_repository_contract_parity" in error + and "[sqlite]" in error + for error in errors + ) + + def test_background_runs_verifier_rejects_missing_locked_scenario( tmp_path: Path, ) -> None: From fb8ca8d49407de977827191ab98af965731ba2f8 Mon Sep 17 00:00:00 2001 From: Merengues0x2A Date: Tue, 14 Jul 2026 16:56:25 +0800 Subject: [PATCH 4/4] fix: harden background run pre-landing contracts --- antelab/runs/manager.py | 19 ++- antelab/runs/sqlite_repository.py | 5 +- antelab/runs/worker.py | 63 ++++----- ...round-runs-local-prelanding-repair.goal.md | 72 ++++++++++ ...d-runs-local-prelanding-repair-round-01.md | 28 ++++ ...d-runs-local-prelanding-repair-round-02.md | 27 ++++ ...d-runs-local-prelanding-repair-round-03.md | 59 ++++++++ ...ound-runs-local-prelanding-repair.STATE.md | 38 +++++ tests/runs/test_durable_adapters.py | 38 +++++ tests/runs/test_experiment_runs.py | 95 +++++++++++++ tests/runs/test_run_worker.py | 130 ++++++++++++++++++ 11 files changed, 538 insertions(+), 36 deletions(-) create mode 100644 goals/background-runs-local-prelanding-repair.goal.md create mode 100644 reports/goals/background-runs-local-prelanding-repair-round-01.md create mode 100644 reports/goals/background-runs-local-prelanding-repair-round-02.md create mode 100644 reports/goals/background-runs-local-prelanding-repair-round-03.md create mode 100644 state/background-runs-local-prelanding-repair.STATE.md diff --git a/antelab/runs/manager.py b/antelab/runs/manager.py index 2bcc60e..53ea7a8 100644 --- a/antelab/runs/manager.py +++ b/antelab/runs/manager.py @@ -2,15 +2,22 @@ from __future__ import annotations +import logging from collections.abc import Callable from uuid import uuid4 from antelab.artifacts import RunArtifact from antelab.runs.artifacts import ArtifactRepository -from antelab.runs.errors import ArtifactNotReadyError +from antelab.runs.errors import ArtifactIntegrityError, ArtifactNotReadyError from antelab.runs.model import IdempotencyKey, RunId, RunRequest, RunSnapshot from antelab.runs.repository import RunRepository +LOGGER = logging.getLogger(__name__) + + +def _escape_incident_field(value: str) -> str: + return value.encode("unicode_escape").decode("ascii") + class ExperimentRuns: def __init__( @@ -49,4 +56,12 @@ def artifact(self, run_id: RunId) -> RunArtifact: raise ArtifactNotReadyError( "artifact is available only for completed runs" ) - return self._artifacts.load(snapshot.artifact) + try: + return self._artifacts.load(snapshot.artifact) + except ArtifactIntegrityError: + LOGGER.error( + "artifact_integrity_incident run_id=%s artifact_id=%s", + _escape_incident_field(run_id), + _escape_incident_field(snapshot.artifact.artifact_id), + ) + raise diff --git a/antelab/runs/sqlite_repository.py b/antelab/runs/sqlite_repository.py index f71a0d0..7c1b026 100644 --- a/antelab/runs/sqlite_repository.py +++ b/antelab/runs/sqlite_repository.py @@ -218,11 +218,14 @@ def _load(self, connection: sqlite3.Connection) -> None: "SELECT payload FROM run_state WHERE singleton = 1" ).fetchone() envelope = cast(dict[str, object], json.loads(cast(str, row[0]))) - if envelope.get("schema_version") not in {1, DURABLE_SCHEMA_VERSION} or set(envelope) != { + schema_version = envelope.get("schema_version") + if schema_version not in {1, DURABLE_SCHEMA_VERSION} or set(envelope) != { "schema_version", "runs", }: raise ValueError("unsupported durable run-state schema") + if schema_version == 1: + raise ValueError("durable schema version 1 is unsupported") raw = cast(dict[str, object], envelope["runs"]) records: dict[RunId, _RunRecord] = {} idempotency: dict[IdempotencyKey, RunId] = {} diff --git a/antelab/runs/worker.py b/antelab/runs/worker.py index a50bdea..bfb8d1f 100644 --- a/antelab/runs/worker.py +++ b/antelab/runs/worker.py @@ -111,33 +111,40 @@ def run_once(self) -> bool: except OSError: return True - def fail_or_cancel( - *, code: str, message: str, retryable: bool - ) -> None: + def acknowledge_cancel_if_requested() -> bool: nonlocal active_lease - current = self._repository.get(active_lease.run_id) - active_lease = replace(active_lease, revision=current.revision) - if current.state == "cancelling": - self._repository.acknowledge_cancel( - active_lease, now=self._now() - ) - return try: - self._repository.fail( - active_lease, - code=code, - message=message, - retryable=retryable, - now=self._now(), - ) - except RunStateConflictError: current = self._repository.get(active_lease.run_id) - if current.state != "cancelling": - raise active_lease = replace(active_lease, revision=current.revision) + if current.state != "cancelling": + return False self._repository.acknowledge_cancel( active_lease, now=self._now() ) + except OSError: + return True + return True + + def fail_or_cancel( + *, code: str, message: str, retryable: bool + ) -> None: + nonlocal active_lease + if acknowledge_cancel_if_requested(): + return + try: + try: + self._repository.fail( + active_lease, + code=code, + message=message, + retryable=retryable, + now=self._now(), + ) + except RunStateConflictError: + if not acknowledge_cancel_if_requested(): + raise + except OSError: + return def persist_progress(tick: int, *, now: datetime) -> None: nonlocal active_lease, last_persisted_at @@ -179,9 +186,7 @@ def progress(tick: int) -> bool: except _TransientRepositoryIOError: return True except _CancelledSignalError: - current = self._repository.get(active_lease.run_id) - active_lease = replace(active_lease, revision=current.revision) - self._repository.acknowledge_cancel(active_lease, now=self._now()) + acknowledge_cancel_if_requested() return True except Exception: fail_or_cancel( @@ -210,12 +215,7 @@ def progress(tick: int) -> bool: except OSError: return True except RunStateConflictError: - if self._repository.get(active_lease.run_id).state == "cancelling": - current = self._repository.get(active_lease.run_id) - active_lease = replace(active_lease, revision=current.revision) - self._repository.acknowledge_cancel( - active_lease, now=self._now() - ) + if acknowledge_cancel_if_requested(): return True raise @@ -225,10 +225,7 @@ def progress(tick: int) -> bool: ) active_lease = replace(active_lease, revision=verifying.revision) except RunStateConflictError: - if self._repository.get(active_lease.run_id).state == "cancelling": - current = self._repository.get(active_lease.run_id) - active_lease = replace(active_lease, revision=current.revision) - self._repository.acknowledge_cancel(active_lease, now=self._now()) + if acknowledge_cancel_if_requested(): return True raise except OSError: diff --git a/goals/background-runs-local-prelanding-repair.goal.md b/goals/background-runs-local-prelanding-repair.goal.md new file mode 100644 index 0000000..788e7da --- /dev/null +++ b/goals/background-runs-local-prelanding-repair.goal.md @@ -0,0 +1,72 @@ +# Goal: Background Runs Local Pre-Landing Repair + +Status: `succeeded` + +- Control shape: finite three-round Goal; concurrency one writer. +- Execution authority: run-to-stop, maximum 3 rounds / 4 hours. +- Workspace: `/private/tmp/antelab-v1-performance-gate-worktree`. +- Exact start ref: `f57da5418ca42de5d66dba35fcc83ec9ccd71b30` on + `feature/v1-performance-gate`. +- Canonical contract: + `docs/superpowers/specs/2026-07-13-background-experiment-runs-architecture.md`. +- State: `state/background-runs-local-prelanding-repair.STATE.md`. +- Reports: `reports/goals/background-runs-local-prelanding-repair-round-NN.md`. + +## Outcome + +Close only four caller-visible pre-landing behaviors established by the local +landing audit: + +1. an engine failure followed by transient repository I/O does not escape + `RunWorker.run_once()`; +2. cancellation acknowledgement followed by transient repository I/O does not + escape `RunWorker.run_once()`; +3. durable schema version 1 is rejected before reconstruction with one stable + pre-release compatibility error; +4. corrupt Published Artifact retrieval raises `ArtifactIntegrityError` and + records a sanitized operational integrity incident. + +## Roles and Rounds + +- Mutator: primary agent, limited to the declared round manifest. +- Evaluator: public-seam pytest contracts plus the existing readiness verifier. +- Independent checker: read-only; cannot change tests, product code, verifier, + contract, or success criteria. +- Controller: this Goal, compact STATE, and append-only round reports. + +Rounds: + +1. RED only. Add behavioral contracts through `RunWorker.run_once()`, durable + `SqliteRunRepository` construction, and `ExperimentRuns.artifact()`. Product + code is protected. Product mutation unlocks only after the independent checker + confirms genuine product RED. +2. GREEN. Make the smallest complete product changes. Do not weaken, delete, + skip, or rewrite the accepted RED contracts. +3. Closure. Run focused, readiness, Ruff, strict mypy, full pytest, diff/scope, + and independent review; record the exact manifest and stop before Git gates. + +## Scope, Verification, and Stops + +- Round 1 allowed: this Goal/STATE/report and + `tests/runs/test_run_worker.py`, `tests/runs/test_durable_adapters.py`, + `tests/runs/test_experiment_runs.py`. +- Round 2 allowed product surface: `antelab/runs/worker.py`, + `antelab/runs/sqlite_repository.py`, and `antelab/runs/manager.py`. +- Round 3 allowed: Goal/STATE/reports plus checker-required corrections within + the already approved test/product surface only. +- Protected: Controller files and historical reports, readiness verifier and + scenario manifests, synchronous CLI/default runner, artifact schema/golden, + dependencies, compact-kernel/headless/roadmap WIP, remote state, deployment, + branches, and worktrees. +- Progress metric: confirmed contracts passing / 4. +- Failure fingerprint: sorted failing focused pytest node IDs. +- No improvement: stop when the same fingerprint repeats across two rounds with + zero newly passing contracts. +- Workload ceiling: no individual simulation above 500 ticks; zero 5,000-tick + executions. +- Stop immediately on scientific artifact/golden drift, default-runner change, + dependency change, remote access, or required work outside the allowed surface. +- Human gates: commit, push, merge, deployment, default-runner switch, 5,000-tick + acceptance, publication, and branch/worktree cleanup. +- Retention: keep the behavioral contracts and final evidence; compact STATE; + do not rewrite historical Controller or Goal reports. diff --git a/reports/goals/background-runs-local-prelanding-repair-round-01.md b/reports/goals/background-runs-local-prelanding-repair-round-01.md new file mode 100644 index 0000000..527de26 --- /dev/null +++ b/reports/goals/background-runs-local-prelanding-repair-round-01.md @@ -0,0 +1,28 @@ +# Background Runs Local Pre-Landing Repair — Round 01 + +Status: `PASS` + +- Workspace: `/private/tmp/antelab-v1-performance-gate-worktree`. +- Start HEAD: `f57da5418ca42de5d66dba35fcc83ec9ccd71b30`. +- Round goal: add only genuine behavioral RED contracts for the four approved + caller-visible behaviors and obtain an independent checker verdict. +- Allowed mutation: Goal/STATE/this report plus three existing test files. +- Product implementation, Controller/history, verifier, CLI/default runner, + dependencies, artifact schema/golden, remote state, and unrelated WIP remain + protected. +- Maximum test workload: 500 ticks; 5,000-tick executions: 0. +- Initial focused RED: 4 / 4 failed for the intended product behaviors. +- Independent checker attempt 1: `FAIL`; the schema-v1 contract matched only an + error substring and did not prove rejection before record reconstruction. +- Test-only correction: the v1 fixture now contains an intentionally + unreconstructable config and requires exact `ValueError` type and message, so + only envelope-first stable rejection can satisfy the contract. +- Corrected focused RED: 4 / 4 failed for the intended product behaviors; no + collection, fixture, environment, or unrelated failure remained. +- Independent checker attempt 2: `PASS`; all four failures are genuine product + RED through the approved public seams. +- Exact Round 1 manifest: this Goal/STATE/report plus + `tests/runs/test_run_worker.py`, `tests/runs/test_durable_adapters.py`, and + `tests/runs/test_experiment_runs.py`. +- `git diff --check`: PASS. Product diff: none. Maximum simulation: 3 ticks. +- Next focus: Round 2 minimal product GREEN within the three authorized files. diff --git a/reports/goals/background-runs-local-prelanding-repair-round-02.md b/reports/goals/background-runs-local-prelanding-repair-round-02.md new file mode 100644 index 0000000..c73ba13 --- /dev/null +++ b/reports/goals/background-runs-local-prelanding-repair-round-02.md @@ -0,0 +1,27 @@ +# Background Runs Local Pre-Landing Repair — Round 02 + +Status: `PASS` + +- Unlock: Round 1 independent checker PASS on four genuine product REDs. +- Allowed product surface: `antelab/runs/worker.py`, + `antelab/runs/sqlite_repository.py`, and `antelab/runs/manager.py` only. +- Accepted tests and verifier are protected from weakening or rewriting. +- Round goal: make the four focused contracts GREEN with the smallest complete + changes while preserving lease recovery, scientific artifacts, and public + caller behavior. +- Maximum test workload: 500 ticks; 5,000-tick executions: 0. +- Product changes: + - worker failure/cancellation terminal writes treat repository `OSError` as a + recoverable attempt interruption and leave the durable lease/state intact; + - durable schema v1 is rejected at the envelope boundary with exact stable + `ValueError` text before any Run record reconstruction; + - corrupt Published Artifact retrieval logs one sanitized operational incident + containing Run ID and content-addressed Artifact ID, then re-raises the + original `ArtifactIntegrityError`. +- Locked focused contracts: 4 / 4 PASS. +- Affected-file regression: 29 / 29 PASS. +- Maximum simulation: 500 ticks in the pre-existing cancellation contract; new + tests use at most 3 ticks. Real 5,000-tick executions: 0. +- Accepted tests/verifier changes during GREEN: none. +- Next focus: Round 3 aggregate gates, adversarial checker, exact manifest, and + stop before all Git/remote/release gates. diff --git a/reports/goals/background-runs-local-prelanding-repair-round-03.md b/reports/goals/background-runs-local-prelanding-repair-round-03.md new file mode 100644 index 0000000..25ca364 --- /dev/null +++ b/reports/goals/background-runs-local-prelanding-repair-round-03.md @@ -0,0 +1,59 @@ +# Background Runs Local Pre-Landing Repair — Round 03 + +Status: `PASS` + +- Scope: local aggregate verification, adversarial review, exact manifest, and + stop-state only. +- Product/test mutation is locked unless the independent checker identifies a + concrete defect inside the already approved surface. +- Initial evidence: focused 4 / 4 PASS; affected-file regression 29 / 29 PASS. +- Required local gates: readiness verifier, Ruff, strict mypy, full pytest, + `git diff --check`, protected-surface proof, and independent checker. +- Test ceiling: at most 500 ticks; real 5,000-tick executions: 0. +- Remote access/CI, commit, push, merge, default-runner switch, deployment, + publication, and branch/worktree cleanup remain forbidden. +- Initial aggregate evidence before adversarial review: readiness 51 / 51, + Ruff PASS, strict mypy 32 source files PASS, full pytest 607 / 607, and + `git diff --check` PASS. +- Initial independent checker: `FAIL`. It found two cancellation acknowledgement + calls inside conflict handlers whose transient `OSError` escaped + `RunWorker.run_once()`, plus raw CR/LF-capable Run/Artifact IDs in the + operational incident log. +- Checker-required behavioral correction, still inside the approved surface: + - two conflict-race tests first failed with escaping `OSError`, then locked + post-engine and begin-verifying cancellation acknowledgement plus + lease-expiry recovery to `cancelled`; + - one malicious-ID test first failed on raw CR/LF, then locked ASCII escaping + of Run/Artifact IDs and exclusion of exception text; + - worker acknowledgement now has one `OSError`-safe helper; incident retrieval + still uses bare `raise` for the original `ArtifactIntegrityError`. +- Final focused contracts: 7 / 7 PASS. Final affected-file regression: 32 / 32 + PASS. Readiness verifier: 54 / 54 PASS and emitted its contract lock message. +- Canonical static gates: Ruff PASS; strict mypy PASS across 32 source files. + A broader non-canonical mypy invocation including `tests` was discarded after + mypy reported the readiness script under two module names; the Makefile/CI + command `mypy antelab scripts` passed. +- Full pytest: 610 / 610 PASS in 35.68 seconds. `git diff --check`: PASS. +- Independent closure checker after corrections: `PASS` with high confidence; + its independent focused 7 / 7, affected 32 / 32, and diff checks passed. +- Exact 11-file manifest: + 1. `antelab/runs/manager.py` + 2. `antelab/runs/sqlite_repository.py` + 3. `antelab/runs/worker.py` + 4. `tests/runs/test_durable_adapters.py` + 5. `tests/runs/test_experiment_runs.py` + 6. `tests/runs/test_run_worker.py` + 7. `goals/background-runs-local-prelanding-repair.goal.md` + 8. `state/background-runs-local-prelanding-repair.STATE.md` + 9. `reports/goals/background-runs-local-prelanding-repair-round-01.md` + 10. `reports/goals/background-runs-local-prelanding-repair-round-02.md` + 11. `reports/goals/background-runs-local-prelanding-repair-round-03.md` +- Protected-surface diff: empty for Controller/history, readiness verifier and + scenarios, synchronous CLI/default runner, artifact schema/golden, + dependencies, compact-kernel/headless/roadmap WIP, and remote/deploy surfaces. +- Controller reconciliation remains a separate, unperformed task; Round 01/02 + historical reports were not rewritten as current facts. +- Maximum executed simulation: 500 ticks in the existing suite; new tests use at + most 3 ticks. Real 5,000-tick executions: 0. +- Stop: no remote access or CI, commit, push, merge, default-runner switch, + deployment, publication, cleanup, or branch/worktree mutation was performed. diff --git a/state/background-runs-local-prelanding-repair.STATE.md b/state/background-runs-local-prelanding-repair.STATE.md new file mode 100644 index 0000000..07b54f0 --- /dev/null +++ b/state/background-runs-local-prelanding-repair.STATE.md @@ -0,0 +1,38 @@ +# State: Background Runs Local Pre-Landing Repair + +Schema version: 2 + +- Status: `succeeded`. +- Round: 3 / 3. +- Execution authority: exhausted; stopped before every human Git/remote gate. +- Workspace: `/private/tmp/antelab-v1-performance-gate-worktree`. +- Branch: `feature/v1-performance-gate`. +- Start HEAD: `f57da5418ca42de5d66dba35fcc83ec9ccd71b30`. +- Start dirty state: clean. +- Product mutation: unlocked by Round 1 independent checker PASS. +- Progress: 4 / 4 confirmed contracts passing. +- Failure fingerprint: + `engine_failure_terminal_io+cancel_ack_terminal_io+schema_v1_envelope_rejection+artifact_integrity_incident`. +- Consecutive no improvement: 0 / 2. +- Test workload: at most 500 ticks; real 5,000-tick executions forbidden. +- Protected: Controller/history, verifier/scenario manifests, synchronous CLI, + default runner, artifact schema/golden, dependencies, remote state, deployment, + unrelated WIP, branches, and worktrees. +- Round 1 evidence: focused 4 / 4 genuine product RED; independent checker PASS + on attempt 2; product diff none; maximum simulation 3 ticks. +- Round 2 evidence: locked focused 4 / 4 PASS; affected-file regression 29 / 29 + PASS; accepted tests and verifier unchanged during GREEN. +- Round 3 checker correction: the initial closure checker found two uncovered + cancel-race acknowledgement paths and control-character injection in incident + IDs. Three added behavioral tests were RED before the shared minimal fix and + GREEN afterward; lease-expiry recovery and ASCII escaping are now locked. +- Final evidence: focused contracts 7 / 7; affected-file regression 32 / 32; + readiness 54 / 54; Ruff PASS; strict mypy 32 source files PASS; full pytest + 610 / 610; diff/scope checks PASS; independent closure checker PASS. +- Exact manifest: 11 files: three product files, three test files, this Goal, + this STATE, and Round 01-03 reports. +- Controller reconciliation: not performed; Controller files and historical + reports remain protected with no diff. +- Next focus: human review of the exact manifest and a separately authorized + scoped Git gate. Remote CI remains outside this Goal. +- Updated: `2026-07-14T07:48:41Z`, writer `primary`. diff --git a/tests/runs/test_durable_adapters.py b/tests/runs/test_durable_adapters.py index ddc0638..b1b5c87 100644 --- a/tests/runs/test_durable_adapters.py +++ b/tests/runs/test_durable_adapters.py @@ -1,3 +1,4 @@ +import json import sqlite3 from collections.abc import Callable from dataclasses import replace @@ -7,6 +8,7 @@ import pytest import antelab.runs.model as run_model +import antelab.runs.sqlite_repository as sqlite_repository_module from antelab.artifacts import RunArtifact from antelab.core.config import load_config from antelab.core.simulation import Simulation @@ -73,6 +75,42 @@ def test_run_request_version_binding_survives_adapter_reconstruction( assert restored.digest() == created.request_digest +def test_schema_v1_is_rejected_before_version_drift_reconstruction( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + database = tmp_path / "schema-v1.sqlite3" + repository = SqliteRunRepository(database) + repository.create_idempotent( + _request(), + IdempotencyKey("legacy-version-bound"), + lambda: RunId("legacy-run"), + ) + with sqlite3.connect(database) as connection: + payload = json.loads( + connection.execute( + "SELECT payload FROM run_state WHERE singleton = 1" + ).fetchone()[0] + ) + payload["schema_version"] = 1 + for record in payload["runs"].values(): + record.pop("engine_version") + record.pop("artifact_schema_version") + record["config"] = {"must_not_be_reconstructed": True} + connection.execute( + "UPDATE run_state SET payload = ? WHERE singleton = 1", + (json.dumps(payload),), + ) + monkeypatch.setattr( + sqlite_repository_module, "__version__", "99.0.0-test-upgrade" + ) + + with pytest.raises(ValueError) as error: + SqliteRunRepository(database) + + assert type(error.value) is ValueError + assert str(error.value) == "durable schema version 1 is unsupported" + + def test_sqlite_lock_contention_is_recoverable_at_worker_boundary( tmp_path: Path, ) -> None: diff --git a/tests/runs/test_experiment_runs.py b/tests/runs/test_experiment_runs.py index 900939c..1394484 100644 --- a/tests/runs/test_experiment_runs.py +++ b/tests/runs/test_experiment_runs.py @@ -1,10 +1,13 @@ +import logging from dataclasses import replace +from datetime import UTC, datetime, timedelta from pathlib import Path import pytest from antelab.core.config import ConfigError, load_config from antelab.runs import ( + ArtifactIntegrityError, ExperimentRuns, IdempotencyConflictError, InMemoryArtifactRepository, @@ -13,6 +16,9 @@ RunNotFoundError, RunRequest, ) +from antelab.runs.artifacts import FilesystemArtifactRepository +from antelab.runs.model import ArtifactRef +from antelab.runs.worker import InProcessSimulationEngine, RunWorker def _runs() -> ExperimentRuns: @@ -84,3 +90,92 @@ def generate_id() -> str: assert generated == [] with pytest.raises(RunNotFoundError): runs.get(RunId("invalid-run")) + + +def test_corrupt_artifact_records_sanitized_operational_incident( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + repository = InMemoryRunRepository() + artifacts = FilesystemArtifactRepository(tmp_path / "artifacts") + runs = ExperimentRuns(repository, artifacts, id_generator=lambda: "incident-run") + submitted = runs.submit(_request(1), idempotency_key="integrity-incident") + worker = RunWorker( + repository, + artifacts, + InProcessSimulationEngine(), + now=lambda: datetime(2026, 7, 14, tzinfo=UTC), + token_factory=lambda: "incident-fence", + lease_duration=timedelta(seconds=10), + ) + assert worker.run_once() is True + completed = runs.get(submitted.run_id) + assert completed.artifact is not None + artifact_path = tmp_path / "artifacts" / completed.artifact.artifact_id + artifact_path.write_bytes(b"corrupt") + + with ( + caplog.at_level(logging.ERROR, logger="antelab.runs.manager"), + pytest.raises(ArtifactIntegrityError), + ): + runs.artifact(submitted.run_id) + + assert [record.getMessage() for record in caplog.records] == [ + "artifact_integrity_incident " + f"run_id={submitted.run_id} artifact_id={completed.artifact.artifact_id}" + ] + assert str(tmp_path) not in caplog.text + + +def test_integrity_incident_escapes_control_characters( + caplog: pytest.LogCaptureFixture, +) -> None: + class MaliciousPublishedArtifactRepository(InMemoryArtifactRepository): + reference = ArtifactRef( + artifact_id="sha256/good\r\nforged_artifact=true", + sha256="0" * 64, + byte_size=1, + schema_version=1, + ) + + def stage(self, artifact): # type: ignore[no-untyped-def] + artifact.validate() + return self.reference + + def publish(self, reference): # type: ignore[no-untyped-def] + assert reference == self.reference + return reference + + def load(self, reference): # type: ignore[no-untyped-def] + assert reference == self.reference + raise ArtifactIntegrityError("secret\nforged_exception=true") + + repository = InMemoryRunRepository() + artifacts = MaliciousPublishedArtifactRepository() + runs = ExperimentRuns( + repository, + artifacts, + id_generator=lambda: "incident\r\nforged_run=true", + ) + submitted = runs.submit(_request(1), idempotency_key="malicious-incident") + worker = RunWorker( + repository, + artifacts, + InProcessSimulationEngine(), + now=lambda: datetime(2026, 7, 14, tzinfo=UTC), + token_factory=lambda: "incident-fence", + lease_duration=timedelta(seconds=10), + ) + assert worker.run_once() is True + + with ( + caplog.at_level(logging.ERROR, logger="antelab.runs.manager"), + pytest.raises(ArtifactIntegrityError), + ): + runs.artifact(submitted.run_id) + + assert [record.getMessage() for record in caplog.records] == [ + "artifact_integrity_incident " + "run_id=incident\\r\\nforged_run=true " + "artifact_id=sha256/good\\r\\nforged_artifact=true" + ] + assert "forged_exception" not in caplog.text diff --git a/tests/runs/test_run_worker.py b/tests/runs/test_run_worker.py index 6745f5e..6eccbc9 100644 --- a/tests/runs/test_run_worker.py +++ b/tests/runs/test_run_worker.py @@ -313,3 +313,133 @@ def record_progress(self, lease, *, tick, now): # type: ignore[no-untyped-def] waiting = runs.get(submitted.run_id) assert waiting.state == "running" assert waiting.failure is None + + +def test_engine_failure_with_transient_terminal_write_waits_for_recovery() -> None: + class TransientFailureRepository(InMemoryRunRepository): + def fail(self, *args, **kwargs): # type: ignore[no-untyped-def] + del args, kwargs + raise OSError("temporary terminal state I/O") + + class FailingEngine: + def execute( + self, request: RunRequest, progress: Callable[[int], bool] + ) -> RunArtifact: + del request, progress + raise RuntimeError("engine failure before terminal persistence") + + repository = TransientFailureRepository() + artifacts = InMemoryArtifactRepository() + runs = ExperimentRuns(repository, artifacts, id_generator=lambda: "run-1") + worker = RunWorker( + repository, + artifacts, + FailingEngine(), + now=lambda: NOW, + token_factory=lambda: "fence-1", + lease_duration=timedelta(seconds=10), + ) + submitted = runs.submit(_request(), idempotency_key="transient-failure-write") + + assert worker.run_once() is True + waiting = runs.get(submitted.run_id) + assert waiting.state == "running" + assert waiting.failure is None + recovered = repository.recover_expired(now=NOW + timedelta(seconds=11)) + assert recovered[0].state == "queued" + + +def test_cancel_ack_with_transient_terminal_write_waits_for_recovery() -> None: + class TransientCancelRepository(InMemoryRunRepository): + def acknowledge_cancel(self, *args, **kwargs): # type: ignore[no-untyped-def] + del args, kwargs + raise OSError("temporary cancellation state I/O") + + repository = TransientCancelRepository() + artifacts = InMemoryArtifactRepository() + runs = ExperimentRuns(repository, artifacts, id_generator=lambda: "run-1") + submitted = runs.submit(_request(), idempotency_key="transient-cancel-write") + + def cancel_on_first_tick(tick: int) -> None: + if tick == 1: + runs.cancel(submitted.run_id) + + worker = RunWorker( + repository, + artifacts, + DeterministicFakeEngine( + _artifact(), ticks=(1, 2, 3), on_tick=cancel_on_first_tick + ), + now=lambda: NOW, + token_factory=lambda: "fence-1", + lease_duration=timedelta(seconds=10), + ) + + assert worker.run_once() is True + waiting = runs.get(submitted.run_id) + assert waiting.state == "cancelling" + assert waiting.artifact is None + recovered = repository.recover_expired(now=NOW + timedelta(seconds=11)) + assert recovered[0].state == "cancelled" + + +def test_post_engine_cancel_ack_transient_io_waits_for_recovery() -> None: + class PostEngineCancelRepository(InMemoryRunRepository): + def record_progress(self, lease, *, tick, now): # type: ignore[no-untyped-def] + self.cancel(lease.run_id) + return super().record_progress(lease, tick=tick, now=now) + + def acknowledge_cancel(self, *args, **kwargs): # type: ignore[no-untyped-def] + del args, kwargs + raise OSError("temporary post-engine cancellation I/O") + + repository = PostEngineCancelRepository() + artifacts = InMemoryArtifactRepository() + runs = ExperimentRuns(repository, artifacts, id_generator=lambda: "run-1") + submitted = runs.submit( + _request(), idempotency_key="post-engine-cancel-write" + ) + worker = RunWorker( + repository, + artifacts, + DeterministicFakeEngine(_artifact(), ticks=()), + now=lambda: NOW, + token_factory=lambda: "fence-1", + lease_duration=timedelta(seconds=10), + ) + + assert worker.run_once() is True + assert runs.get(submitted.run_id).state == "cancelling" + recovered = repository.recover_expired(now=NOW + timedelta(seconds=11)) + assert recovered[0].state == "cancelled" + + +def test_begin_verifying_cancel_ack_transient_io_waits_for_recovery() -> None: + class BeginVerifyingCancelRepository(InMemoryRunRepository): + def begin_verifying(self, lease, *, now): # type: ignore[no-untyped-def] + self.cancel(lease.run_id) + return super().begin_verifying(lease, now=now) + + def acknowledge_cancel(self, *args, **kwargs): # type: ignore[no-untyped-def] + del args, kwargs + raise OSError("temporary verifying cancellation I/O") + + repository = BeginVerifyingCancelRepository() + artifacts = InMemoryArtifactRepository() + runs = ExperimentRuns(repository, artifacts, id_generator=lambda: "run-1") + submitted = runs.submit( + _request(), idempotency_key="verifying-cancel-write" + ) + worker = RunWorker( + repository, + artifacts, + DeterministicFakeEngine(_artifact(), ticks=(1, 2, 3)), + now=lambda: NOW, + token_factory=lambda: "fence-1", + lease_duration=timedelta(seconds=10), + ) + + assert worker.run_once() is True + assert runs.get(submitted.run_id).state == "cancelling" + recovered = repository.recover_expired(now=NOW + timedelta(seconds=11)) + assert recovered[0].state == "cancelled"