From fbeb85dbb24709c575d1cc05932986551c6bd32b Mon Sep 17 00:00:00 2001 From: jeremydixon22 Date: Mon, 10 Aug 2026 05:14:01 -0400 Subject: [PATCH 1/2] test(python): qualify measured local first use --- .../python-runtime-qualification.yml | 53 +++- PUBLIC-EXPORT-MANIFEST.json | 22 +- README.md | 5 + runtimes/python/README.md | 24 +- .../src/vyral_runtime/_local_experience.py | 25 ++ .../python/tests/test_local_experience.py | 17 ++ .../test-python-runtime-platform-matrix.py | 37 +++ scripts/verify-python-runtime-install.py | 267 +++++++++++++++++- .../verify-python-runtime-platform-matrix.py | 75 +++++ scripts/verify-python-runtime.sh | 9 +- .../write-python-runtime-platform-receipt.py | 162 ++++++++++- 11 files changed, 663 insertions(+), 33 deletions(-) diff --git a/.github/workflows/python-runtime-qualification.yml b/.github/workflows/python-runtime-qualification.yml index 4804485..0433657 100644 --- a/.github/workflows/python-runtime-qualification.yml +++ b/.github/workflows/python-runtime-qualification.yml @@ -4,12 +4,14 @@ on: workflow_dispatch: inputs: baseline_runtime_wheel_url: - description: HTTPS URL for the previously qualified 0.1.x runtime wheel - required: true + description: Optional HTTPS URL for a previously qualified 0.1.x runtime wheel + required: false + default: "" type: string baseline_runtime_wheel_sha256: - description: Expected SHA-256 for the baseline runtime wheel - required: true + description: Required SHA-256 when a baseline runtime wheel URL is supplied + required: false + default: "" type: string permissions: @@ -58,8 +60,8 @@ jobs: - name: Build and consume clean artifacts run: | python -m build --sdist --wheel --outdir artifacts/python-runtime runtimes/python - python scripts/verify-python-runtime-install.py --server artifacts/python-runtime - python scripts/write-python-runtime-platform-receipt.py artifacts/python-runtime --output artifacts/qualification/platform.json + python scripts/verify-python-runtime-install.py --server artifacts/python-runtime --output artifacts/qualification/clean-install.json + python scripts/write-python-runtime-platform-receipt.py artifacts/python-runtime --clean-install-evidence artifacts/qualification/clean-install.json --output artifacts/qualification/platform.json - name: Upload platform qualification evidence uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -100,7 +102,37 @@ jobs: npm ci --ignore-scripts --prefix clients/javascript (cd clients/javascript && npm pack --json --pack-destination ../../artifacts/javascript) + - name: Validate optional upgrade baseline inputs + id: baseline + env: + BASELINE_RUNTIME_WHEEL_URL: ${{ inputs.baseline_runtime_wheel_url }} + BASELINE_RUNTIME_WHEEL_SHA256: ${{ inputs.baseline_runtime_wheel_sha256 }} + run: | + python - <<'PY' + import os + + url_supplied = bool(os.environ["BASELINE_RUNTIME_WHEEL_URL"].strip()) + hash_supplied = bool( + os.environ["BASELINE_RUNTIME_WHEEL_SHA256"].strip() + ) + if url_supplied != hash_supplied: + raise SystemExit( + "Supply both baseline wheel inputs or leave both empty." + ) + with open( + os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8" + ) as output: + output.write( + "enabled=" + ("true" if url_supplied else "false") + "\n" + ) + print( + "python-runtime-upgrade-baseline=" + + ("supplied" if url_supplied else "not-supplied") + ) + PY + - name: Fetch verified upgrade baseline + if: ${{ steps.baseline.outputs.enabled == 'true' }} env: BASELINE_RUNTIME_WHEEL_URL: ${{ inputs.baseline_runtime_wheel_url }} BASELINE_RUNTIME_WHEEL_SHA256: ${{ inputs.baseline_runtime_wheel_sha256 }} @@ -150,9 +182,16 @@ jobs: scripts/verify-python-runtime-external-worker.sh artifacts/runtime/*.whl scripts/verify-python-runtime-mcp-conformance.sh artifacts/runtime/*.whl python scripts/verify-python-runtime-security.py artifacts/runtime/*.whl --output artifacts/qualification/python-runtime-security.json - python scripts/verify-python-runtime-upgrade.py artifacts/baseline/*.whl artifacts/runtime/*.whl --output artifacts/qualification/python-runtime-upgrade.json python scripts/benchmark-python-runtime.py --output artifacts/qualification/python-runtime-performance.json + - name: Run installed upgrade qualification + if: ${{ steps.baseline.outputs.enabled == 'true' }} + run: >- + python scripts/verify-python-runtime-upgrade.py + artifacts/baseline/*.whl + artifacts/runtime/*.whl + --output artifacts/qualification/python-runtime-upgrade.json + - name: Upload qualification evidence if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/PUBLIC-EXPORT-MANIFEST.json b/PUBLIC-EXPORT-MANIFEST.json index 9a56d98..6e18ff9 100644 --- a/PUBLIC-EXPORT-MANIFEST.json +++ b/PUBLIC-EXPORT-MANIFEST.json @@ -74,7 +74,7 @@ { "mode": "644", "path": ".github/workflows/python-runtime-qualification.yml", - "sha256": "66fa2c00be266319be8a9a64a85e98901f493868c2280d3627fe5ab7a89b7cf9" + "sha256": "e2644bbf1ea1b5cf06917e8a2f0a51359ebc519823fab7309ac7be6652f56d86" }, { "mode": "644", @@ -124,7 +124,7 @@ { "mode": "644", "path": "README.md", - "sha256": "49c08cc55f653a1987a7ce9daf93a6a2cf2742af674363c9c479bb7f0622f5a9" + "sha256": "e0725bb5a4e54f6143f1b1afbbae3e48131ffc1ed7671ea7b261529dae01e6b7" }, { "mode": "644", @@ -654,7 +654,7 @@ { "mode": "644", "path": "runtimes/python/README.md", - "sha256": "d6f49d16ef0d5c26e262488d6c0a62dbb1edc1653091f216ba410ca39313f8c0" + "sha256": "edec7886576e96ca62c04d80952efef731bc3ec4d6bc88dc8f3dace1dbee04a7" }, { "mode": "644", @@ -789,7 +789,7 @@ { "mode": "644", "path": "runtimes/python/src/vyral_runtime/_local_experience.py", - "sha256": "7ed47623ae40fef2164a6ee94552a402470d33d99b0c3ed5b2ec5d9cc83e467f" + "sha256": "c00ff042ebb73e6518b9f2e87a50c61a4b1daf6512dd6fab472403f3d794689d" }, { "mode": "644", @@ -1189,7 +1189,7 @@ { "mode": "644", "path": "runtimes/python/tests/test_local_experience.py", - "sha256": "ec153cd5f316be04bdeb2d2c38ffbbcda22af771a80478cadeff483d03bbc848" + "sha256": "fe7948aa0501f626ca7b7485048d0ee3a22438f187c8857256ab5773037664c0" }, { "mode": "644", @@ -1514,7 +1514,7 @@ { "mode": "644", "path": "scripts/test-python-runtime-platform-matrix.py", - "sha256": "6ec2270241fad474e45ef221232ec000c8215309e410e10230d145e6a8368a45" + "sha256": "3f8d53916d72ad2bfe25d7ba45d64baafa272d9b1f13661fc797021e2116a26f" }, { "mode": "755", @@ -1694,7 +1694,7 @@ { "mode": "755", "path": "scripts/verify-python-runtime-install.py", - "sha256": "f64d7f981402da9718265b310a30a8ac3d8a8f15e5f6f71997fde3ea01d53a58" + "sha256": "2a105eee7f5c5b648c3160b2e5f4d3e88c710053a19b672b617454397d0ee132" }, { "mode": "755", @@ -1704,7 +1704,7 @@ { "mode": "644", "path": "scripts/verify-python-runtime-platform-matrix.py", - "sha256": "082accbd32ed53d170be35af3299c64d2d6ba228f7d3af0eda0e6f7c0e349235" + "sha256": "479ebe8be702cdec6f2f6a2ca1cfaa8d2f67c1affacd7c0c6c93ba1fececdd4d" }, { "mode": "644", @@ -1719,7 +1719,7 @@ { "mode": "755", "path": "scripts/verify-python-runtime.sh", - "sha256": "3b958588166ac6ad09a4f426203a99c36a104c58fb00a45d92e1aee879f613b2" + "sha256": "6a0c28f9829a74e5d4adfaac6a0c6832d532a2e14826f992cd17c7cf8728cf8f" }, { "mode": "755", @@ -1744,7 +1744,7 @@ { "mode": "644", "path": "scripts/write-python-runtime-platform-receipt.py", - "sha256": "74c8a3bbd6f539ea02127ef2f506da4a07a9b0967ac69bd553fb38b4ac4f375a" + "sha256": "4610d63939fb59ece2a9747bbd70bf501139b5223aa5683a2a5c00ed039dd167" }, { "mode": "644", @@ -4074,5 +4074,5 @@ ], "schemaVersion": 1, "sourceDirty": false, - "treeSha256": "8a21a0a511b3acdce8fb6bfa727627d19b46e00a2666f14c036341366aa7100c" + "treeSha256": "4ab971e5a620e67910e568268aac9cfb41897f88420840c1c487a5dcbf40cfdf" } diff --git a/README.md b/README.md index 229d48e..ebcb3e0 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,11 @@ without rerunning the quickstart: vyral-runtime inspect --root ./.vyral/quickstart ``` +Add `--json` to capture machine-readable first-citation, durable-receipt, +restart-recovery, and completion timings. Clean wheel and source-distribution +qualification runs the same command in an isolated environment and enforces a +five-minute first-citation budget rather than relying on a synthetic smoke. + The quickstart owns only the directory bearing its marker. Its reset command refuses foreign, symbolic-link, home, filesystem-root, and working-directory targets: diff --git a/runtimes/python/README.md b/runtimes/python/README.md index 5b50269..071f212 100644 --- a/runtimes/python/README.md +++ b/runtimes/python/README.md @@ -92,6 +92,13 @@ pre-release path. Once a qualified wheel is published, the install command can be replaced with `python -m pip install vyral-runtime` without changing the quickstart or inspection commands. +The quickstart JSON includes measured `firstCitationMs`, `durableReceiptMs`, +`restartRecoveryMs`, and `completedMs` milestones. Artifact qualification runs +the installed wheel and sdist through the first command, a second-process +idempotent replay, independent inspection, and marker-bounded reset. Each +supported platform cell rejects a first citation or complete command taking +more than five minutes. + ```python from vyral_runtime import VyralRuntime @@ -252,16 +259,21 @@ dotnet test tests/Vyral.Tests.Conformance/Vyral.Tests.Conformance.csproj \ ``` The main verification command runs the full unit suite with branch -instrumentation using pinned Coverage.py 7.15.2 and enforces a 77.5% combined +instrumentation using pinned Coverage.py 7.15.4 and enforces a 77.5% combined line/branch regression floor. The floor is a regression guard, not a claim that every dispatch or error branch is exhaustively tested. The cross-platform qualification workflow is manual-only: `.github/workflows/python-runtime-qualification.yml`. This preserves the repository's current GitHub-run gate while keeping the promotion matrix -reproducible. Dispatch requires the HTTPS URL and SHA-256 of a previously -qualified `0.1.x` wheel so the installed upgrade gate cannot silently -self-compare the candidate. The aggregate job then runs +reproducible. Every cell builds both artifacts, installs each into a clean +environment, completes the real cited/restart quickstart, replays it from a +second process, inspects its limitations, and safely resets its owned state. +The HTTPS URL and SHA-256 of a previously qualified `0.1.x` wheel are optional +paired inputs: when supplied, the workflow also runs the installed upgrade +gate, which cannot silently self-compare the candidate. The aggregate job then runs `scripts/verify-python-runtime-platform-matrix.py`; all nine cells must be -clean, refer to one commit, and agree on runtime and contract identity before -the matrix is valid promotion evidence. +clean, refer to one commit, agree on runtime and contract identity, and carry +passing measured local-experience evidence before the matrix is valid. A run +without a baseline proves the platform matrix but is not upgrade evidence and +does not by itself authorize maturity promotion. diff --git a/runtimes/python/src/vyral_runtime/_local_experience.py b/runtimes/python/src/vyral_runtime/_local_experience.py index 8455046..77aea16 100644 --- a/runtimes/python/src/vyral_runtime/_local_experience.py +++ b/runtimes/python/src/vyral_runtime/_local_experience.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from pathlib import Path import shutil +from time import perf_counter from .embeddings import EmbeddingProviderRegistry from .execution import ( @@ -109,6 +110,10 @@ class LocalQuickstartResult: completed_status: str completed_result: object dispatched_runs: int + first_citation_ms: float + durable_receipt_ms: float + restart_recovery_ms: float + completed_ms: float def to_dict(self) -> dict[str, object]: return { @@ -143,6 +148,12 @@ def to_dict(self) -> dict[str, object]: "completedResult": self.completed_result, "dispatchedRuns": self.dispatched_runs, }, + "timings": { + "firstCitationMs": self.first_citation_ms, + "durableReceiptMs": self.durable_receipt_ms, + "restartRecoveryMs": self.restart_recovery_ms, + "completedMs": self.completed_ms, + }, } @@ -180,6 +191,7 @@ async def run_local_quickstart( *, emit: Callable[[str], None] | None = None, ) -> LocalQuickstartResult: + started_at = perf_counter() root = prepare_local_quickstart_root(root_path) created_chunks = 0 reused_chunks = 0 @@ -288,6 +300,7 @@ async def run_local_quickstart( raise RuntimeError( "The local quickstart did not produce citation-ready context." ) + first_citation_ms = _elapsed_ms(started_at) citations = tuple( LocalQuickstartCitation( citation.source_label or citation.record_id, @@ -318,6 +331,7 @@ async def run_local_quickstart( raise RuntimeError( "Durable admission did not preserve the idempotent run identity." ) + durable_receipt_ms = _elapsed_ms(started_at) _emit( emit, ( @@ -327,6 +341,7 @@ async def run_local_quickstart( ) _emit(emit, "Closed the first runtime instance.") + restart_started_at = perf_counter() with VyralRuntime.open_local( root, execution_plugins=(_QUICKSTART_PLUGIN,), @@ -336,6 +351,7 @@ async def run_local_quickstart( raise RuntimeError( "The admitted run was not present after reopening local state." ) + restart_recovery_ms = _elapsed_ms(restart_started_at) _emit( emit, ( @@ -359,6 +375,7 @@ async def run_local_quickstart( "the same durable identity." ), ) + completed_ms = _elapsed_ms(started_at) return LocalQuickstartResult( root_path=str(root), @@ -383,6 +400,10 @@ async def run_local_quickstart( completed_status=completed.status, completed_result=completed.result, dispatched_runs=dispatched, + first_citation_ms=first_citation_ms, + durable_receipt_ms=durable_receipt_ms, + restart_recovery_ms=restart_recovery_ms, + completed_ms=completed_ms, ) @@ -516,6 +537,10 @@ def _emit(emit: Callable[[str], None] | None, message: str) -> None: emit(message) +def _elapsed_ms(started_at: float) -> float: + return round((perf_counter() - started_at) * 1_000, 3) + + def run_local_quickstart_sync( root_path: str | Path, *, diff --git a/runtimes/python/tests/test_local_experience.py b/runtimes/python/tests/test_local_experience.py index ecdee4c..7589a58 100644 --- a/runtimes/python/tests/test_local_experience.py +++ b/runtimes/python/tests/test_local_experience.py @@ -35,6 +35,23 @@ def test_citations_and_execution_survive_a_real_reopen(self) -> None: self.assertEqual("queued", first.persisted_status) self.assertEqual("succeeded", first.completed_status) self.assertEqual(1, first.dispatched_runs) + self.assertGreaterEqual(first.first_citation_ms, 0) + self.assertGreaterEqual( + first.durable_receipt_ms, + first.first_citation_ms, + ) + self.assertGreaterEqual(first.restart_recovery_ms, 0) + self.assertGreaterEqual( + first.completed_ms, + first.durable_receipt_ms, + ) + timings = first.to_dict()["timings"] + self.assertIsInstance(timings, dict) + assert isinstance(timings, dict) + self.assertEqual( + first.first_citation_ms, + timings["firstCitationMs"], + ) self.assertIsInstance(first.completed_result, dict) assert isinstance(first.completed_result, dict) self.assertEqual( diff --git a/scripts/test-python-runtime-platform-matrix.py b/scripts/test-python-runtime-platform-matrix.py index 297125c..6c9e6cd 100644 --- a/scripts/test-python-runtime-platform-matrix.py +++ b/scripts/test-python-runtime-platform-matrix.py @@ -39,6 +39,20 @@ def _receipt(system: str, python: str) -> dict[str, object]: "fixtureVersion": "1.0.0", "profiles": [{"id": "portable", "available": True}], }, + "localExperience": { + "schemaVersion": "vyral.python-runtime-clean-install.v1", + "status": "passed", + "serverExtraVerified": True, + "firstCitationBudgetMs": 300_000, + "artifacts": [ + { + "artifactKind": artifact_kind, + "firstCommandMs": 1_200, + "firstCitationMs": 900, + } + for artifact_kind in ("wheel", "sdist") + ], + }, "gates": sorted(MODULE.REQUIRED_GATES), } @@ -73,6 +87,29 @@ def main() -> int: pass else: raise SystemExit("A duplicate/incomplete matrix was accepted.") + + over_budget = _receipt("Linux", "3.10") + local_experience = over_budget["localExperience"] + assert isinstance(local_experience, dict) + artifacts = local_experience["artifacts"] + assert isinstance(artifacts, list) + first_artifact = artifacts[0] + assert isinstance(first_artifact, dict) + first_artifact["firstCitationMs"] = 300_001 + over_budget_path = root / "over-budget.json" + over_budget_path.write_text( + json.dumps(over_budget), + encoding="utf-8", + ) + try: + replaced = root / "Linux" / "3.10" / "platform.json" + MODULE.verify( + [over_budget_path, *(path for path in receipts if path != replaced)] + ) + except MODULE.MatrixError: + pass + else: + raise SystemExit("An over-budget local experience was accepted.") print("python-runtime-platform-matrix-test=ok cells=9") return 0 diff --git a/scripts/verify-python-runtime-install.py b/scripts/verify-python-runtime-install.py index 57b7c36..1007d4a 100755 --- a/scripts/verify-python-runtime-install.py +++ b/scripts/verify-python-runtime-install.py @@ -1,16 +1,23 @@ #!/usr/bin/env python3 -"""Verify clean wheel/sdist installation on the current Python platform.""" +"""Verify clean artifacts and the measured local developer path.""" from __future__ import annotations import argparse +from datetime import datetime, timezone +import json from pathlib import Path +import platform import subprocess import sys import tempfile +from time import perf_counter +from typing import Any, Mapping import venv +FIRST_CITATION_BUDGET_MS = 5 * 60 * 1_000 + SMOKE = r""" from vyral_runtime import ( VyralRuntime, @@ -38,19 +45,217 @@ """ +class InstallVerificationError(RuntimeError): + """Raised when an installed artifact cannot complete the golden path.""" + + def _interpreter(environment: Path) -> Path: if sys.platform == "win32": return environment / "Scripts" / "python.exe" return environment / "bin" / "python" -def _command(*arguments: str) -> None: - subprocess.run(arguments, check=True) +def _command( + *arguments: str, + capture_output: bool = False, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + arguments, + check=True, + capture_output=capture_output, + text=True, + ) + + +def _json_command(*arguments: str) -> Mapping[str, Any]: + completed = _command(*arguments, capture_output=True) + try: + value = json.loads(completed.stdout) + except json.JSONDecodeError as error: + raise InstallVerificationError( + f"Command did not return JSON: {' '.join(arguments)}" + ) from error + if not isinstance(value, Mapping): + raise InstallVerificationError("Command JSON must be an object.") + return value -def _install_and_smoke(artifact: Path, destination: Path) -> None: +def _object( + value: object, + label: str, +) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise InstallVerificationError(f"{label} must be an object.") + return value + + +def _number(value: object, label: str) -> float: + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise InstallVerificationError(f"{label} must be numeric.") + selected = float(value) + if selected < 0: + raise InstallVerificationError(f"{label} must not be negative.") + return selected + + +def _quickstart_smoke( + python: Path, + state_root: Path, +) -> dict[str, object]: + started_at = perf_counter() + first = _json_command( + str(python), + "-m", + "vyral_runtime", + "quickstart", + "--root", + str(state_root), + "--json", + ) + first_command_ms = round((perf_counter() - started_at) * 1_000, 3) + retrieval = _object(first.get("retrieval"), "retrieval") + execution = _object(first.get("execution"), "execution") + timings = _object(first.get("timings"), "timings") + citations = retrieval.get("citations") + if not isinstance(citations, list) or not citations: + raise InstallVerificationError( + "The clean local quickstart returned no citations." + ) + first_citation_ms = _number( + timings.get("firstCitationMs"), + "timings.firstCitationMs", + ) + durable_receipt_ms = _number( + timings.get("durableReceiptMs"), + "timings.durableReceiptMs", + ) + completed_ms = _number( + timings.get("completedMs"), + "timings.completedMs", + ) + if first_citation_ms > FIRST_CITATION_BUDGET_MS: + raise InstallVerificationError( + "The clean local quickstart exceeded the five-minute " + f"first-citation budget: {first_citation_ms:.3f} ms." + ) + if first_command_ms > FIRST_CITATION_BUDGET_MS: + raise InstallVerificationError( + "The clean local quickstart command exceeded five minutes: " + f"{first_command_ms:.3f} ms." + ) + if not ( + first_citation_ms <= durable_receipt_ms <= completed_ms + ): + raise InstallVerificationError( + "Quickstart timing milestones are not monotonic." + ) + if ( + execution.get("admittedStatus") != "queued" + or execution.get("persistedStatusAfterReopen") != "queued" + or execution.get("completedStatus") != "succeeded" + or execution.get("dispatchedRuns") != 1 + ): + raise InstallVerificationError( + "The clean quickstart did not prove receipt-before-dispatch " + "and restart completion." + ) + run_id = execution.get("runId") + if not isinstance(run_id, str) or not run_id: + raise InstallVerificationError( + "The clean quickstart did not return a durable run ID." + ) + + replay_started_at = perf_counter() + replay = _json_command( + str(python), + "-m", + "vyral_runtime", + "quickstart", + "--root", + str(state_root), + "--json", + ) + replay_command_ms = round( + (perf_counter() - replay_started_at) * 1_000, + 3, + ) + replay_retrieval = _object(replay.get("retrieval"), "replay.retrieval") + replay_execution = _object(replay.get("execution"), "replay.execution") + if ( + replay_execution.get("runId") != run_id + or replay_execution.get("admissionReplayed") is not True + or replay_execution.get("dispatchedRuns") != 0 + or replay_retrieval.get("createdChunks") != 0 + or replay_retrieval.get("reusedChunks") != 3 + ): + raise InstallVerificationError( + "The second clean-process quickstart did not replay the same " + "run and persisted corpus." + ) + + inspection = _json_command( + str(python), + "-m", + "vyral_runtime", + "inspect", + "--root", + str(state_root), + "--json", + ) + providers = _object(inspection.get("providers"), "inspection.providers") + embeddings = _object( + providers.get("embeddings"), + "inspection.providers.embeddings", + ) + if ( + inspection.get("topology") != "local-single-node" + or embeddings.get("semanticQuality") != "lexical" + or embeddings.get("requiresNetwork") is not False + ): + raise InstallVerificationError( + "Installed-runtime inspection did not disclose the local topology " + "and model-free embedding limitation." + ) + + removed = _json_command( + str(python), + "-m", + "vyral_runtime", + "quickstart", + "--root", + str(state_root), + "--reset", + "--json", + ) + if not isinstance(removed.get("removedRootPath"), str) or state_root.exists(): + raise InstallVerificationError( + "The installed quickstart did not safely remove its owned state." + ) + + return { + "status": "passed", + "citationCount": len(citations), + "firstCommandMs": first_command_ms, + "firstCitationMs": first_citation_ms, + "durableReceiptMs": durable_receipt_ms, + "completedMs": completed_ms, + "replayCommandMs": replay_command_ms, + "receiptBeforeDispatch": True, + "restartPreservedRunIdentity": True, + "secondProcessReplayed": True, + "safeReset": True, + } + + +def _install_and_smoke( + artifact: Path, + destination: Path, + *, + artifact_kind: str, +) -> dict[str, object]: venv.EnvBuilder(with_pip=True, clear=True).create(destination) python = _interpreter(destination) + install_started_at = perf_counter() _command( str(python), "-m", @@ -60,7 +265,15 @@ def _install_and_smoke(artifact: Path, destination: Path) -> None: "--no-deps", str(artifact), ) + install_ms = round((perf_counter() - install_started_at) * 1_000, 3) + quickstart = _quickstart_smoke(python, destination / "quickstart-state") _command(str(python), "-c", SMOKE) + return { + "artifactKind": artifact_kind, + "artifactName": artifact.name, + "installMs": install_ms, + "quickstart": quickstart, + } def _server_smoke(wheel: Path, destination: Path) -> None: @@ -94,6 +307,11 @@ def main() -> int: action="store_true", help="Also install the server extra and exercise the CLI.", ) + parser.add_argument( + "--output", + type=Path, + help="Optional path for a machine-readable clean-install receipt.", + ) arguments = parser.parse_args() root = arguments.artifact_directory.resolve() wheels = sorted(root.glob("vyral_runtime-*.whl")) @@ -107,15 +325,50 @@ def main() -> int: prefix="vyral-runtime-install-" ) as temporary: destination = Path(temporary) - _install_and_smoke(wheels[0], destination / "wheel") - _install_and_smoke(sdists[0], destination / "sdist") + artifact_results = [ + _install_and_smoke( + wheels[0], + destination / "wheel", + artifact_kind="wheel", + ), + _install_and_smoke( + sdists[0], + destination / "sdist", + artifact_kind="sdist", + ), + ] if arguments.server: _server_smoke(wheels[0], destination / "server") + receipt: dict[str, object] = { + "schemaVersion": "vyral.python-runtime-clean-install.v1", + "status": "passed", + "generatedAtUtc": datetime.now(timezone.utc) + .isoformat() + .replace("+00:00", "Z"), + "environment": { + "implementation": sys.implementation.name, + "pythonVersion": platform.python_version(), + "system": platform.system(), + "machine": platform.machine(), + }, + "firstCitationBudgetMs": FIRST_CITATION_BUDGET_MS, + "serverExtraVerified": arguments.server, + "artifacts": artifact_results, + } + if arguments.output is not None: + output = arguments.output.resolve() + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(receipt, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print( "python-runtime-clean-install=ok " f"python={sys.version_info.major}.{sys.version_info.minor} " - f"platform={sys.platform}" + f"platform={sys.platform} artifacts=2 " + f"first-citation-budget-ms={FIRST_CITATION_BUDGET_MS}" ) return 0 diff --git a/scripts/verify-python-runtime-platform-matrix.py b/scripts/verify-python-runtime-platform-matrix.py index d2326ce..a24fe5f 100644 --- a/scripts/verify-python-runtime-platform-matrix.py +++ b/scripts/verify-python-runtime-platform-matrix.py @@ -28,6 +28,13 @@ "clean-wheel-install", "clean-sdist-install", "server-extra-install", + "clean-wheel-local-quickstart", + "clean-sdist-local-quickstart", + "first-citation-under-five-minutes", + "receipt-before-dispatch", + "restart-preserved-run-identity", + "second-process-idempotent-replay", + "owned-state-safe-reset", } @@ -144,6 +151,69 @@ def verify(receipt_paths: list[Path]) -> dict[str, Any]: raise MatrixError( f"Platform receipt {path} does not prove every required gate." ) + local_experience = _object( + root.get("localExperience"), + f"{path}: localExperience", + ) + if ( + local_experience.get("schemaVersion") + != "vyral.python-runtime-clean-install.v1" + or local_experience.get("status") != "passed" + or local_experience.get("serverExtraVerified") is not True + ): + raise MatrixError( + f"Platform receipt {path} has no passing clean-install evidence." + ) + first_citation_budget = local_experience.get( + "firstCitationBudgetMs" + ) + if ( + not isinstance(first_citation_budget, (int, float)) + or isinstance(first_citation_budget, bool) + ): + raise MatrixError( + f"Platform receipt {path} has no first-citation budget." + ) + experience_artifacts = local_experience.get("artifacts") + if ( + not isinstance(experience_artifacts, list) + or len(experience_artifacts) != 2 + ): + raise MatrixError( + f"Platform receipt {path} must prove wheel and sdist quickstarts." + ) + artifact_kinds: set[str] = set() + first_citation_values: list[float] = [] + first_command_values: list[float] = [] + for artifact_value in experience_artifacts: + artifact = _object( + artifact_value, + f"{path}: localExperience artifact", + ) + kind = _text( + artifact.get("artifactKind"), + f"{path}: localExperience artifactKind", + ) + artifact_kinds.add(kind) + first_citation = artifact.get("firstCitationMs") + first_command = artifact.get("firstCommandMs") + if ( + not isinstance(first_citation, (int, float)) + or isinstance(first_citation, bool) + or not isinstance(first_command, (int, float)) + or isinstance(first_command, bool) + or float(first_citation) > float(first_citation_budget) + or float(first_command) > float(first_citation_budget) + ): + raise MatrixError( + f"Platform receipt {path} exceeded its local time budget." + ) + first_citation_values.append(float(first_citation)) + first_command_values.append(float(first_command)) + if artifact_kinds != {"wheel", "sdist"}: + raise MatrixError( + f"Platform receipt {path} has incomplete artifact kinds." + ) cells[cell] = { "system": system, "python": python, @@ -152,6 +222,11 @@ def verify(receipt_paths: list[Path]) -> dict[str, Any]: "platform": environment.get("platform"), "sqliteVersion": environment.get("sqliteVersion"), "fts5Available": environment.get("fts5Available"), + "localExperience": { + "firstCitationBudgetMs": first_citation_budget, + "maxFirstCitationMs": max(first_citation_values), + "maxFirstCommandMs": max(first_command_values), + }, } missing = sorted(EXPECTED_MATRIX - set(cells)) diff --git a/scripts/verify-python-runtime.sh b/scripts/verify-python-runtime.sh index 37e205b..3ab1b1b 100755 --- a/scripts/verify-python-runtime.sh +++ b/scripts/verify-python-runtime.sh @@ -179,6 +179,12 @@ assert first["execution"]["admittedStatus"] == "queued" assert first["execution"]["persistedStatusAfterReopen"] == "queued" assert first["execution"]["completedStatus"] == "succeeded" assert first["execution"]["dispatchedRuns"] == 1 +assert 0 <= first["timings"]["firstCitationMs"] <= 300_000 +assert ( + first["timings"]["firstCitationMs"] + <= first["timings"]["durableReceiptMs"] + <= first["timings"]["completedMs"] +) assert inspection["topology"] == "local-single-node" assert inspection["providers"]["records"]["healthy"] is True assert inspection["providers"]["execution"]["healthy"] is True @@ -190,7 +196,8 @@ assert replay["retrieval"]["reusedChunks"] == 3 print( "python-runtime-local-quickstart=ok " f"run={first['execution']['runId']} " - f"citations={len(first['retrieval']['citations'])}" + f"citations={len(first['retrieval']['citations'])} " + f"first-citation-ms={first['timings']['firstCitationMs']}" ) PY "$work_root/venv/bin/vyral-runtime" quickstart \ diff --git a/scripts/write-python-runtime-platform-receipt.py b/scripts/write-python-runtime-platform-receipt.py index 9c062e2..a7cb372 100644 --- a/scripts/write-python-runtime-platform-receipt.py +++ b/scripts/write-python-runtime-platform-receipt.py @@ -12,7 +12,7 @@ import sqlite3 import subprocess import sys -from typing import Any +from typing import Any, Mapping ROOT = Path(__file__).resolve().parents[1] @@ -51,6 +51,110 @@ def _fts5_available() -> bool: return True +def _mapping(value: object, label: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError(f"{label} must be an object.") + return value + + +def _load_clean_install_evidence(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError( + "Clean-install evidence must be valid UTF-8 JSON." + ) from error + root = _mapping(value, "clean-install evidence") + if ( + root.get("schemaVersion") + != "vyral.python-runtime-clean-install.v1" + or root.get("status") != "passed" + ): + raise ValueError("Clean-install evidence is not a passing v1 receipt.") + environment = _mapping( + root.get("environment"), + "clean-install environment", + ) + if ( + environment.get("system") != platform.system() + or environment.get("pythonVersion") != platform.python_version() + ): + raise ValueError( + "Clean-install evidence was not produced by this platform cell." + ) + budget = root.get("firstCitationBudgetMs") + if not isinstance(budget, (int, float)) or isinstance(budget, bool): + raise ValueError("Clean-install evidence has no numeric time budget.") + artifacts = root.get("artifacts") + if not isinstance(artifacts, list) or len(artifacts) != 2: + raise ValueError( + "Clean-install evidence must contain wheel and sdist results." + ) + summaries: list[dict[str, Any]] = [] + kinds: set[str] = set() + for index, artifact_value in enumerate(artifacts): + artifact = _mapping( + artifact_value, + f"clean-install artifacts[{index}]", + ) + kind = artifact.get("artifactKind") + if not isinstance(kind, str): + raise ValueError("Clean-install artifact kind must be a string.") + kinds.add(kind) + quickstart = _mapping( + artifact.get("quickstart"), + f"clean-install {kind} quickstart", + ) + first_citation_ms = quickstart.get("firstCitationMs") + first_command_ms = quickstart.get("firstCommandMs") + if ( + not isinstance(first_citation_ms, (int, float)) + or isinstance(first_citation_ms, bool) + or not isinstance(first_command_ms, (int, float)) + or isinstance(first_command_ms, bool) + or float(first_citation_ms) > float(budget) + or float(first_command_ms) > float(budget) + ): + raise ValueError( + f"Clean-install {kind} exceeded the first-use time budget." + ) + if any( + quickstart.get(field) is not True + for field in ( + "receiptBeforeDispatch", + "restartPreservedRunIdentity", + "secondProcessReplayed", + "safeReset", + ) + ): + raise ValueError( + f"Clean-install {kind} did not prove the complete local path." + ) + summaries.append( + { + "artifactKind": kind, + "artifactName": artifact.get("artifactName"), + "installMs": artifact.get("installMs"), + "firstCommandMs": first_command_ms, + "firstCitationMs": first_citation_ms, + "durableReceiptMs": quickstart.get("durableReceiptMs"), + "completedMs": quickstart.get("completedMs"), + } + ) + if kinds != {"wheel", "sdist"}: + raise ValueError( + "Clean-install evidence must prove one wheel and one sdist." + ) + return { + "schemaVersion": root.get("schemaVersion"), + "status": root.get("status"), + "sha256": _hash(path), + "firstCitationBudgetMs": budget, + "serverExtraVerified": root.get("serverExtraVerified") is True, + "artifacts": sorted(summaries, key=lambda item: str(item["artifactKind"])), + } + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument( @@ -64,6 +168,14 @@ def main() -> int: default="portable-matrix", choices=("portable-matrix", "local-rehearsal"), ) + parser.add_argument( + "--clean-install-evidence", + type=Path, + help=( + "Passing clean wheel/sdist quickstart receipt. Required for " + "portable-matrix evidence." + ), + ) arguments = parser.parse_args() artifact_root = arguments.artifact_directory.resolve() wheels = sorted(artifact_root.glob("vyral_runtime-*.whl")) @@ -72,6 +184,40 @@ def main() -> int: parser.error( "artifact directory must contain exactly one runtime wheel and sdist" ) + if ( + arguments.scope == "portable-matrix" + and arguments.clean_install_evidence is None + ): + parser.error( + "--clean-install-evidence is required for portable-matrix receipts" + ) + clean_install = ( + _load_clean_install_evidence( + arguments.clean_install_evidence.resolve() + ) + if arguments.clean_install_evidence is not None + else None + ) + if clean_install is not None: + clean_artifacts = clean_install.get("artifacts") + assert isinstance(clean_artifacts, list) + clean_names = { + str(item.get("artifactName")) + for item in clean_artifacts + if isinstance(item, Mapping) + } + expected_names = {wheels[0].name, sdists[0].name} + if clean_names != expected_names: + parser.error( + "clean-install evidence does not describe the supplied artifacts" + ) + if ( + arguments.scope == "portable-matrix" + and clean_install.get("serverExtraVerified") is not True + ): + parser.error( + "portable-matrix clean-install evidence must verify the server extra" + ) readiness = VyralRuntime().readiness() if readiness.status != "ok" or readiness.contract is None: @@ -134,6 +280,7 @@ def main() -> int: } for path in (*wheels, *sdists) ], + "localExperience": clean_install, "gates": [ "contract-sync", "language-neutral-conformance", @@ -145,6 +292,19 @@ def main() -> int: "clean-wheel-install", "clean-sdist-install", "server-extra-install", + *( + ( + "clean-wheel-local-quickstart", + "clean-sdist-local-quickstart", + "first-citation-under-five-minutes", + "receipt-before-dispatch", + "restart-preserved-run-identity", + "second-process-idempotent-replay", + "owned-state-safe-reset", + ) + if clean_install is not None + else () + ), ], } output = arguments.output.resolve() From 046f863eeb49764475e2be2447041861775f4458 Mon Sep 17 00:00:00 2001 From: jeremydixon22 Date: Mon, 10 Aug 2026 05:22:58 -0400 Subject: [PATCH 2/2] fix(python): make local qualification portable on Windows --- .gitattributes | 1 + PUBLIC-EXPORT-MANIFEST.json | 17 ++-- .../src/vyral_runtime/local/object_store.py | 86 ++++++++++++++++--- runtimes/python/tests/test_object_store.py | 63 +++++++++++++- scripts/export-public-tree.py | 2 + scripts/test-export-public-tree.py | 2 + 6 files changed, 152 insertions(+), 19 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/PUBLIC-EXPORT-MANIFEST.json b/PUBLIC-EXPORT-MANIFEST.json index 6e18ff9..872cf7c 100644 --- a/PUBLIC-EXPORT-MANIFEST.json +++ b/PUBLIC-EXPORT-MANIFEST.json @@ -1,5 +1,5 @@ { - "fileCount": 814, + "fileCount": 815, "files": [ { "mode": "644", @@ -16,6 +16,11 @@ "path": ".gcloudignore", "sha256": "8407c8a13ba547c7b2ec31aaf4ca0ac79264ef510972cbc865f688609ec48f56" }, + { + "mode": "644", + "path": ".gitattributes", + "sha256": "d60f352d0db1404c70afb4bb8b2ca3fd1c610572aa40720e8a0b7baa7885418c" + }, { "mode": "644", "path": ".github/CODEOWNERS", @@ -1019,7 +1024,7 @@ { "mode": "644", "path": "runtimes/python/src/vyral_runtime/local/object_store.py", - "sha256": "de5562d037fb22a189dd644c9c4eec10a522c027c6da593f7a46ac104716df4c" + "sha256": "03c6a2d8319d9d86adb67f9a1633a5201a39ef7f29684957b30c8fd5ced53690" }, { "mode": "644", @@ -1204,7 +1209,7 @@ { "mode": "644", "path": "runtimes/python/tests/test_object_store.py", - "sha256": "9c335616e93c44ae80e8822ee9fb6907e375252464a4cfe92733651e92585b17" + "sha256": "b149ca5e551e41c4b42b62667e7c114f88b93a0bb824dc47a1ae8c0939857537" }, { "mode": "644", @@ -1399,7 +1404,7 @@ { "mode": "755", "path": "scripts/export-public-tree.py", - "sha256": "be4a9d5d9bba838554c2abe803b0c84f3e66f4d8a970c5c31695df4be2271b76" + "sha256": "309b2a397b85609ea2dd1ecc03b06662e40cbe615cba88c3fb505ee25b6bb306" }, { "mode": "755", @@ -1504,7 +1509,7 @@ { "mode": "644", "path": "scripts/test-export-public-tree.py", - "sha256": "a52a5ed8f2b9a89cb18a2292c3121f40c51f2cd3f12833c8d1c17b10217df8d0" + "sha256": "e834af5d61ce44dcd2ddcd3f0cfab37710aa30ed080122d41a6491427babe8c6" }, { "mode": "755", @@ -4074,5 +4079,5 @@ ], "schemaVersion": 1, "sourceDirty": false, - "treeSha256": "4ab971e5a620e67910e568268aac9cfb41897f88420840c1c487a5dcbf40cfdf" + "treeSha256": "24e6d0d2e0c1d04ee2b6149149cf2fa42f3e9bdd329bcea9db9f26c9d3297791" } diff --git a/runtimes/python/src/vyral_runtime/local/object_store.py b/runtimes/python/src/vyral_runtime/local/object_store.py index cfffbd7..469abe4 100644 --- a/runtimes/python/src/vyral_runtime/local/object_store.py +++ b/runtimes/python/src/vyral_runtime/local/object_store.py @@ -4,22 +4,47 @@ from dataclasses import dataclass, field from datetime import datetime, timezone from hashlib import sha256 +from importlib import import_module from io import BytesIO import json import os from pathlib import Path import tempfile from threading import Lock, RLock -from typing import Any, BinaryIO, Mapping +from typing import Any, BinaryIO, Mapping, Protocol, cast from .._datetime import parse_iso_datetime from ..async_runtime import RuntimeExecutor from .models import JSONObject -try: - import fcntl -except ImportError: # pragma: no cover - the supported baseline is POSIX. - fcntl = None # type: ignore[assignment] +class _FcntlModule(Protocol): + LOCK_EX: int + LOCK_UN: int + + def flock(self, file_descriptor: int, operation: int) -> None: ... + + +class _MsvcrtModule(Protocol): + LK_LOCK: int + LK_UNLCK: int + + def locking( + self, + file_descriptor: int, + mode: int, + byte_count: int, + ) -> None: ... + + +def _optional_module(name: str) -> Any | None: + try: + return import_module(name) + except ModuleNotFoundError: + return None + + +_fcntl = cast(_FcntlModule | None, _optional_module("fcntl")) +_msvcrt = cast(_MsvcrtModule | None, _optional_module("msvcrt")) DEFAULT_OBJECT_LIST_LIMIT = 100 @@ -199,10 +224,12 @@ def __enter__(self) -> _FileLock: self._thread_lock.acquire() try: self._file = self._path.open("a+b") - if fcntl is not None: - fcntl.flock(self._file.fileno(), fcntl.LOCK_EX) + _lock_process_file(self._file) return self except BaseException: + if self._file is not None: + self._file.close() + self._file = None self._thread_lock.release() raise @@ -212,11 +239,46 @@ def __exit__( exc: object, traceback: object, ) -> None: - if self._file is not None: - if fcntl is not None: - fcntl.flock(self._file.fileno(), fcntl.LOCK_UN) - self._file.close() - self._thread_lock.release() + try: + if self._file is not None: + _unlock_process_file(self._file) + finally: + try: + if self._file is not None: + self._file.close() + self._file = None + finally: + self._thread_lock.release() + + +def _lock_process_file(file: BinaryIO) -> None: + if _fcntl is not None: + _fcntl.flock(file.fileno(), _fcntl.LOCK_EX) + return + if _msvcrt is not None: + file.seek(0, os.SEEK_END) + if file.tell() == 0: + file.write(b"\0") + file.flush() + file.seek(0) + _msvcrt.locking(file.fileno(), _msvcrt.LK_LOCK, 1) + return + raise RuntimeError( + "The local object store has no supported process-locking backend." + ) + + +def _unlock_process_file(file: BinaryIO) -> None: + if _fcntl is not None: + _fcntl.flock(file.fileno(), _fcntl.LOCK_UN) + return + if _msvcrt is not None: + file.seek(0) + _msvcrt.locking(file.fileno(), _msvcrt.LK_UNLCK, 1) + return + raise RuntimeError( + "The local object store has no supported process-locking backend." + ) class FileObjectStore: diff --git a/runtimes/python/tests/test_object_store.py b/runtimes/python/tests/test_object_store.py index 974119b..e9b7a8b 100644 --- a/runtimes/python/tests/test_object_store.py +++ b/runtimes/python/tests/test_object_store.py @@ -3,8 +3,11 @@ from io import BytesIO from pathlib import Path from tempfile import TemporaryDirectory -from threading import Barrier, Thread +from threading import Barrier, RLock, Thread import unittest +from unittest.mock import patch + +import vyral_runtime.local.object_store as object_store from vyral_runtime import ( FileObjectStore, @@ -230,6 +233,64 @@ def write(value: bytes) -> None: self.assertEqual(["rejected", "written"], sorted(outcomes)) + def test_windows_process_lock_backend_locks_one_stable_byte(self) -> None: + class FakeMsvcrt: + LK_LOCK = 1 + LK_UNLCK = 2 + + def __init__(self) -> None: + self.calls: list[tuple[int, int]] = [] + + def locking( + self, + file_descriptor: int, + mode: int, + byte_count: int, + ) -> None: + del file_descriptor + self.calls.append((mode, byte_count)) + + fake = FakeMsvcrt() + path = Path(self.temporary_directory.name) / "windows.lock" + with ( + patch.object(object_store, "_fcntl", None), + patch.object(object_store, "_msvcrt", fake), + object_store._FileLock(path, RLock()), + ): + self.assertEqual(b"\0", path.read_bytes()) + + self.assertEqual([(fake.LK_LOCK, 1), (fake.LK_UNLCK, 1)], fake.calls) + + def test_process_lock_failure_closes_file_and_releases_thread_lock( + self, + ) -> None: + class FailingMsvcrt: + LK_LOCK = 1 + LK_UNLCK = 2 + + def locking( + self, + file_descriptor: int, + mode: int, + byte_count: int, + ) -> None: + del file_descriptor, mode, byte_count + raise OSError("lock unavailable") + + path = Path(self.temporary_directory.name) / "failure.lock" + thread_lock = RLock() + with ( + patch.object(object_store, "_fcntl", None), + patch.object(object_store, "_msvcrt", FailingMsvcrt()), + self.assertRaisesRegex(OSError, "lock unavailable"), + ): + with object_store._FileLock(path, thread_lock): + self.fail("The failing lock backend unexpectedly entered.") + + self.assertTrue(thread_lock.acquire(blocking=False)) + thread_lock.release() + path.unlink() + if __name__ == "__main__": unittest.main() diff --git a/scripts/export-public-tree.py b/scripts/export-public-tree.py index b32045b..78930f8 100755 --- a/scripts/export-public-tree.py +++ b/scripts/export-public-tree.py @@ -20,6 +20,7 @@ ROOT_FILES = { ".dockerignore", + ".gitattributes", ".gcloudignore", ".gitignore", "CODE_OF_CONDUCT.md", @@ -123,6 +124,7 @@ } REQUIRED_FILES = { + ".gitattributes", ".github/workflows/ci.yml", ".github/workflows/python-runtime-qualification.yml", "conformance/invariant.md", diff --git a/scripts/test-export-public-tree.py b/scripts/test-export-public-tree.py index 4cb3c13..7a5212a 100644 --- a/scripts/test-export-public-tree.py +++ b/scripts/test-export-public-tree.py @@ -71,6 +71,8 @@ def main() -> int: if not MODULE.is_public("ROADMAP.md"): raise SystemExit("The public roadmap is absent from the export.") + if not MODULE.is_public(".gitattributes"): + raise SystemExit("The cross-platform line-ending policy is absent from the export.") print( "public-export-policy-test=ok " f"docs={len(public_docs)} designs={len(public_designs)} "