From 58992020c1fb2bd76476cd84642b14cd9f36a672 Mon Sep 17 00:00:00 2001 From: M9nx Date: Fri, 9 Oct 2026 02:43:15 +0300 Subject: [PATCH 1/6] fix(test): make openlabs setup envelope test hermetic without docker --- scripts/test_openlabs_cli.py | 55 ++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/scripts/test_openlabs_cli.py b/scripts/test_openlabs_cli.py index 5def683c..af458782 100644 --- a/scripts/test_openlabs_cli.py +++ b/scripts/test_openlabs_cli.py @@ -17,9 +17,9 @@ sys.path.insert(0, str(SCRIPTS)) from cli_contract import validate_envelope # noqa: E402 -from openlabs_cli.app import run_cli # noqa: E402 from openlabs_cli.context import CliContext # noqa: E402 from openlabs_cli.envelope import build_envelope, Diagnostic, CliResult # noqa: E402 +from openlabs_cli.environment_probe import CheckResult, EnvironmentReport # noqa: E402 from openlabs_cli.root import find_repo_root # noqa: E402 @@ -74,14 +74,62 @@ def test_unknown_command_json() -> None: def test_setup_json_envelope() -> None: - code, out, _err = _run(["--json", "setup"]) - payload = _validate_json(out, label="setup.json") + report = EnvironmentReport( + checks=( + CheckResult(name="repo_root", ok=True, detail=str(REPO_ROOT)), + CheckResult(name="python", ok=True, detail="3.12"), + ), + tier="tier1", + runtime="docker", + missing=(), + guidance=(), + ) + ctx = CliContext(repo_root=REPO_ROOT, json_mode=True) + with mock.patch( + "openlabs_cli.commands.setup.run_environment_probe", + return_value=report, + ): + from openlabs_cli.commands.setup import handle_setup # noqa: E402 + + result = handle_setup(ctx, []) + envelope = build_envelope(result, ctx) + payload = envelope + errors = validate_envelope(payload, path="setup.json") + assert not errors assert payload["command"] == "setup" assert payload["data"]["action"] == "run" assert "checks" in payload["data"] assert "tier" in payload["data"] +def test_setup_missing_docker_diagnostic() -> None: + report = EnvironmentReport( + checks=( + CheckResult( + name="docker_client", + ok=False, + detail="missing", + diagnostic_key="environment.docker.client_missing", + ), + ), + tier="unsupported", + runtime="unknown", + missing=("docker_client",), + guidance=("install Docker Engine",), + ) + ctx = CliContext(repo_root=REPO_ROOT, json_mode=True) + with mock.patch( + "openlabs_cli.commands.setup.run_environment_probe", + return_value=report, + ): + from openlabs_cli.commands.setup import handle_setup # noqa: E402 + + result = handle_setup(ctx, []) + envelope = build_envelope(result, ctx) + assert envelope["ok"] is False + assert envelope["diagnostics"][0]["key"] == "environment.docker.client_missing" + + def test_repo_layout_invalid() -> None: with tempfile.TemporaryDirectory() as tmp: code, out, err = _run(["--json", "setup"], cwd=Path(tmp)) @@ -163,6 +211,7 @@ def main() -> int: test_help_json_envelope, test_unknown_command_json, test_setup_json_envelope, + test_setup_missing_docker_diagnostic, test_repo_layout_invalid, test_dry_run_flag_reaches_envelope, test_redaction_in_diagnostic_message, From 06aa2f0b11b4337f14038ad9539c4b5e0c87ee96 Mon Sep 17 00:00:00 2001 From: M9nx Date: Fri, 9 Oct 2026 02:43:15 +0300 Subject: [PATCH 2/6] feat(ci): add evidence artifact helper and tier1 lifecycle runner --- .gitignore | 1 + scripts/evidence_artifact.py | 130 ++++++++++++++++ scripts/run_m2_tier1_lifecycle.py | 240 ++++++++++++++++++++++++++++++ 3 files changed, 371 insertions(+) create mode 100644 scripts/evidence_artifact.py create mode 100644 scripts/run_m2_tier1_lifecycle.py diff --git a/.gitignore b/.gitignore index f1cc2830..585c3810 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ tsconfig.tsbuildinfo # GitHub API cache for the lab sync. Regenerates automatically. .cache/ .openlabs/ +.tmp-test-artifacts/ /.codegraph/ diff --git a/scripts/evidence_artifact.py b/scripts/evidence_artifact.py new file mode 100644 index 00000000..747380d7 --- /dev/null +++ b/scripts/evidence_artifact.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Bounded, redacted JSON evidence for CI lifecycle artifacts.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any + +from diagnostic_registry import redact_mapping + +DEFAULT_MAX_BYTES = 256 * 1024 +FLAG_PLAINTEXT_RE = re.compile(r"duck\{[a-z0-9_]{16,40}\}") +EVIDENCE_VERSION = "openlabs.evidence.v1" + + +def prepare_evidence_document(data: dict[str, Any]) -> dict[str, Any]: + prepared = redact_mapping(dict(data)) + if isinstance(prepared, dict): + meta = prepared.get("meta") + if not isinstance(meta, dict): + meta = {} + prepared["meta"] = meta + meta["redacted"] = True + return prepared # type: ignore[return-value] + + +def _shrink_for_limit(data: dict[str, Any], *, max_bytes: int) -> tuple[dict[str, Any], int, bool]: + current = prepare_evidence_document(data) + encoded = (json.dumps(current, indent=2) + "\n").encode("utf-8") + original_bytes = len(encoded) + if original_bytes <= max_bytes: + return current, original_bytes, False + + working = dict(current) + steps = working.get("steps") + if isinstance(steps, list): + while len(steps) > 1: + steps.pop() + working["truncated"] = True + working["original_bytes"] = original_bytes + encoded = (json.dumps(working, indent=2) + "\n").encode("utf-8") + if len(encoded) <= max_bytes: + working["written_bytes"] = len(encoded) + return working, original_bytes, True + working["steps"] = steps[:1] if steps else [] + + working.pop("raw_stdout", None) + working.pop("raw_stderr", None) + working["truncated"] = True + working["original_bytes"] = original_bytes + encoded = (json.dumps(working, indent=2) + "\n").encode("utf-8") + if len(encoded) > max_bytes: + minimal = { + "version": working.get("version", EVIDENCE_VERSION), + "truncated": True, + "original_bytes": original_bytes, + "written_bytes": max_bytes, + "detail": "evidence exceeded size limit after truncation", + "meta": {"redacted": True}, + } + encoded = (json.dumps(minimal, indent=2) + "\n").encode("utf-8") + if len(encoded) > max_bytes: + encoded = encoded[: max_bytes - 1] + if not encoded.endswith(b"\n"): + encoded += b"\n" + working = json.loads(encoded.decode("utf-8", errors="replace")) + if not isinstance(working, dict): + working = minimal + else: + working["written_bytes"] = len(encoded) + return working, original_bytes, True + + +def write_bounded_json(path: Path, data: dict[str, Any], *, max_bytes: int = DEFAULT_MAX_BYTES) -> None: + bounded, _original, _truncated = _shrink_for_limit(data, max_bytes=max_bytes) + text = json.dumps(bounded, indent=2) + "\n" + path.write_text(text, encoding="utf-8") + + +def assert_no_plaintext_secrets(path: Path) -> None: + text = path.read_text(encoding="utf-8") + matches = FLAG_PLAINTEXT_RE.findall(text) + if matches: + print( + f"evidence artifact {path.name}: found {len(matches)} plaintext flag pattern(s)", + file=sys.stderr, + ) + raise SystemExit(1) + + +def bound_file(source: Path, dest: Path, *, max_bytes: int = DEFAULT_MAX_BYTES) -> None: + data = json.loads(source.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError("evidence source must be a JSON object") + wrapped = { + "version": EVIDENCE_VERSION, + "source": source.name, + **data, + } + write_bounded_json(dest, wrapped, max_bytes=max_bytes) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + bound = sub.add_parser("bound", help="read raw JSON, write bounded redacted copy") + bound.add_argument("source", type=Path) + bound.add_argument("dest", type=Path) + bound.add_argument("--max-bytes", type=int, default=DEFAULT_MAX_BYTES) + + scan = sub.add_parser("scan-secrets", help="fail if plaintext flags appear in file") + scan.add_argument("path", type=Path) + + args = parser.parse_args() + if args.command == "bound": + bound_file(args.source, args.dest, max_bytes=args.max_bytes) + return 0 + if args.command == "scan-secrets": + assert_no_plaintext_secrets(args.path) + return 0 + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_m2_tier1_lifecycle.py b/scripts/run_m2_tier1_lifecycle.py new file mode 100644 index 00000000..a6ad1da5 --- /dev/null +++ b/scripts/run_m2_tier1_lifecycle.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""Run Tier 1 duck-cross lifecycle via openlabs and emit bounded evidence.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import time +from pathlib import Path +from typing import Any, Callable + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from cli_contract import validate_envelope # noqa: E402 +from evidence_artifact import EVIDENCE_VERSION, write_bounded_json # noqa: E402 +from openlabs_cli.compose_ops import compose_argv_from_state # noqa: E402 +from openlabs_cli.context import CliContext # noqa: E402 +from openlabs_cli.lab_discovery import resolve_lab # noqa: E402 +from openlabs_cli.lab_runtime import run_compose # noqa: E402 +from openlabs_cli.runner import SubprocessCommandRunner # noqa: E402 +from openlabs_cli.state_store import load_state # noqa: E402 + +LAB = "duck-cross" +STEP_TIMEOUT_S = 900.0 +OPENLABS = REPO_ROOT / "openlabs" + +LIFECYCLE_STEPS: tuple[tuple[str, list[str]], ...] = ( + ("lab_setup", ["lab", "setup", LAB]), + ("lab_verify", ["lab", "verify", LAB]), + ("lab_reset", ["lab", "reset", LAB]), + ("lab_start", ["lab", "start", LAB]), + ("lab_status", ["lab", "status", LAB]), + ("lab_stop", ["lab", "stop", LAB]), +) + + +def _global_argv(tail: list[str]) -> list[str]: + return [str(OPENLABS), "--json", "--non-interactive", *tail] + + +def run_openlabs_step( + argv: list[str], + *, + step_name: str, + cwd: Path, + timeout: float = STEP_TIMEOUT_S, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, +) -> dict[str, Any]: + started = time.time() + if runner is None: + try: + completed = subprocess.run( + argv, + cwd=cwd, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except FileNotFoundError: + return { + "name": step_name, + "ok": False, + "exit_code": 127, + "seconds": round(time.time() - started, 2), + "diagnostic_key": "environment.docker.client_missing", + "detail": "openlabs executable or docker client unavailable", + "stdout": "", + "stderr": "", + } + except subprocess.TimeoutExpired: + return { + "name": step_name, + "ok": False, + "exit_code": 124, + "seconds": round(time.time() - started, 2), + "diagnostic_key": "lifecycle.cli.internal", + "detail": f"command timed out after {timeout}s", + "stdout": "", + "stderr": "", + } + else: + try: + completed = runner(argv, cwd=cwd, timeout=timeout) + except FileNotFoundError: + return { + "name": step_name, + "ok": False, + "exit_code": 127, + "seconds": round(time.time() - started, 2), + "diagnostic_key": "environment.docker.client_missing", + "detail": "openlabs executable or docker client unavailable", + "stdout": "", + "stderr": "", + } + except subprocess.TimeoutExpired: + return { + "name": step_name, + "ok": False, + "exit_code": 124, + "seconds": round(time.time() - started, 2), + "diagnostic_key": "lifecycle.cli.internal", + "detail": f"command timed out after {timeout}s", + "stdout": "", + "stderr": "", + } + + stdout = completed.stdout or "" + stderr = completed.stderr or "" + record: dict[str, Any] = { + "name": step_name, + "ok": completed.returncode == 0, + "exit_code": completed.returncode, + "seconds": round(time.time() - started, 2), + "stdout": stdout.strip(), + "stderr": stderr.strip(), + } + if stdout.strip(): + try: + envelope = json.loads(stdout) + record["envelope"] = envelope + errors = validate_envelope(envelope, path="tier1-step.json") + if errors: + record["ok"] = False + record["envelope_errors"] = errors + record["detail"] = "invalid openlabs.command.v1 envelope" + elif isinstance(envelope, dict) and not envelope.get("ok", False): + record["ok"] = False + diagnostics = envelope.get("diagnostics") or [] + if diagnostics and isinstance(diagnostics[0], dict): + record["diagnostic_key"] = diagnostics[0].get("key") + record["detail"] = "openlabs returned ok false" + except json.JSONDecodeError: + record["ok"] = False + record["detail"] = "stdout was not valid JSON" + elif completed.returncode != 0: + record["detail"] = stderr.strip() or "command failed with no JSON stdout" + return record + + +def _cleanup(ctx: CliContext) -> dict[str, Any]: + started = time.time() + steps: list[dict[str, Any]] = [] + ok = True + detail = "cleanup complete" + selection = resolve_lab(ctx.repo_root, LAB) + if not hasattr(selection, "lab_dir"): + return { + "ok": False, + "detail": "duck-cross not found for cleanup", + "seconds": round(time.time() - started, 2), + "steps": steps, + } + state = load_state(ctx.repo_root, LAB) + if state: + try: + argv = compose_argv_from_state( + ctx.repo_root, + lab_dir=selection.lab_dir, + state=state, + subcommand=["down", "--remove-orphans"], + ) + code, out = run_compose(ctx, argv, timeout=120.0) + steps.append({"action": "compose_down", "ok": code == 0, "detail": out[:200]}) + if code != 0: + ok = False + detail = "compose down during cleanup failed" + except OSError as exc: + ok = False + detail = str(exc) + reset_argv = _global_argv(["lab", "reset", LAB]) + reset = run_openlabs_step(reset_argv, step_name="cleanup_reset", cwd=ctx.repo_root, timeout=120.0) + steps.append({"action": "lab_reset", **{k: reset.get(k) for k in ("ok", "exit_code", "detail")}}) + if not reset.get("ok", False): + ok = False + detail = "lab reset during cleanup failed" + return { + "ok": ok, + "detail": detail, + "seconds": round(time.time() - started, 2), + "steps": steps, + } + + +def run_tier1_lifecycle( + *, + repo_root: Path, + output: Path | None, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, +) -> tuple[int, dict[str, Any]]: + steps: list[dict[str, Any]] = [] + failed = False + ctx = CliContext(repo_root=repo_root, runner=SubprocessCommandRunner()) + cleanup: dict[str, Any] = {"ok": True, "detail": "not run", "seconds": 0.0, "steps": []} + document: dict[str, Any] = { + "version": EVIDENCE_VERSION, + "lab": LAB, + "steps": steps, + "cleanup": cleanup, + "truncated": False, + "meta": {"redacted": True}, + } + try: + for name, tail in LIFECYCLE_STEPS: + record = run_openlabs_step( + _global_argv(tail), + step_name=name, + cwd=repo_root, + runner=runner, + ) + record["step"] = name + steps.append(record) + if not record.get("ok", False): + failed = True + finally: + cleanup = _cleanup(ctx) + document["steps"] = steps + document["cleanup"] = cleanup + if output is not None: + write_bounded_json(output, document) + + if not cleanup.get("ok", True): + failed = True + + return (1 if failed else 0), document + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--json", type=Path, required=True, help="write bounded evidence JSON") + args = parser.parse_args() + code, _doc = run_tier1_lifecycle(repo_root=REPO_ROOT, output=args.json) + return code + + +if __name__ == "__main__": + sys.exit(main()) From 395296622861c1b886e90a4d5b9a7beca706749b Mon Sep 17 00:00:00 2001 From: M9nx Date: Fri, 9 Oct 2026 02:43:15 +0300 Subject: [PATCH 3/6] test(ci): add tier1 lifecycle runner unit gates --- scripts/test_run_m2_tier1_lifecycle.py | 235 +++++++++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 scripts/test_run_m2_tier1_lifecycle.py diff --git a/scripts/test_run_m2_tier1_lifecycle.py b/scripts/test_run_m2_tier1_lifecycle.py new file mode 100644 index 00000000..6b70a620 --- /dev/null +++ b/scripts/test_run_m2_tier1_lifecycle.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Non-Docker tests for run_m2_tier1_lifecycle and evidence_artifact.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from unittest import mock + +REPO_ROOT = Path(__file__).resolve().parent.parent +SCRIPTS = REPO_ROOT / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +from cli_contract import validate_envelope # noqa: E402 +from diagnostic_registry import FLAG_PLAINTEXT_RE # noqa: E402 +from evidence_artifact import ( # noqa: E402 + assert_no_plaintext_secrets, + bound_file, + write_bounded_json, +) +import run_m2_tier1_lifecycle as tier1_mod # noqa: E402 +from run_m2_tier1_lifecycle import ( # noqa: E402 + LIFECYCLE_STEPS, + run_openlabs_step, + run_tier1_lifecycle, +) + +def _temp_json(name: str) -> Path: + directory = REPO_ROOT / ".tmp-test-artifacts" + directory.mkdir(exist_ok=True) + return directory / name + + +OK_ENVELOPE = { + "version": "openlabs.command.v1", + "command": "lab", + "ok": True, + "exit_code": 0, + "dry_run": False, + "data": {"action": "status", "lab": "duck-cross"}, + "diagnostics": [], + "meta": {"redacted": False}, +} + + +def _completed(code: int, stdout: str = "", stderr: str = "") -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(args=[], returncode=code, stdout=stdout, stderr=stderr) + + +def test_success_sequence_all_steps() -> None: + calls: list[list[str]] = [] + + def runner(argv: list[str], *, cwd: Path, timeout: float) -> subprocess.CompletedProcess[str]: + _ = cwd, timeout + calls.append(list(argv)) + return _completed(0, json.dumps(OK_ENVELOPE)) + + with mock.patch.object( + tier1_mod, + "_cleanup", + return_value={"ok": True, "detail": "ok", "seconds": 0.0, "steps": []}, + ): + out = _temp_json("success-evidence.json") + code, doc = run_tier1_lifecycle(repo_root=REPO_ROOT, output=out, runner=runner) + assert code == 0 + assert len(doc["steps"]) == len(LIFECYCLE_STEPS) + assert all(step["ok"] for step in doc["steps"]) + assert calls[0][:4] == [str(REPO_ROOT / "openlabs"), "--json", "--non-interactive", "lab"] + + +def test_command_failure_partial_evidence() -> None: + def runner(argv: list[str], *, cwd: Path, timeout: float) -> subprocess.CompletedProcess[str]: + _ = cwd, timeout + if "setup" in argv: + return _completed(1, "", "setup failed") + return _completed(0, json.dumps(OK_ENVELOPE)) + + with mock.patch.object( + tier1_mod, + "_cleanup", + return_value={"ok": True, "detail": "ok", "seconds": 0.0, "steps": []}, + ): + out = _temp_json("failure-evidence.json") + code, doc = run_tier1_lifecycle(repo_root=REPO_ROOT, output=out, runner=runner) + assert code != 0 + assert doc["steps"][0]["ok"] is False + assert out.is_file() + json.loads(out.read_text(encoding="utf-8")) + + +def test_malformed_json_from_cli() -> None: + record = run_openlabs_step( + [str(REPO_ROOT / "openlabs"), "--json", "--non-interactive", "lab", "status", "duck-cross"], + step_name="lab_status", + cwd=REPO_ROOT, + runner=lambda argv, cwd, timeout: _completed(0, "not-json", ""), + ) + assert record["ok"] is False + assert "not valid JSON" in record["detail"] + + +def test_invalid_envelope_recorded() -> None: + bad = {"version": "wrong", "command": "lab", "ok": True, "exit_code": 0, "dry_run": False, "data": {}, "diagnostics": []} + record = run_openlabs_step( + [str(REPO_ROOT / "openlabs"), "--json", "--non-interactive", "lab", "list"], + step_name="lab_list", + cwd=REPO_ROOT, + runner=lambda argv, cwd, timeout: _completed(0, json.dumps(bad), ""), + ) + assert record["ok"] is False + assert record.get("envelope_errors") + + +def test_secret_redaction_in_artifact() -> None: + secret = "duck{abcdefghijklmnopqrstuvwxyz}" + path = _temp_json("redact-out.json") + write_bounded_json( + path, + {"version": "openlabs.evidence.v1", "steps": [{"detail": secret}]}, + max_bytes=64 * 1024, + ) + text = path.read_text(encoding="utf-8") + assert not FLAG_PLAINTEXT_RE.search(text) + assert_no_plaintext_secrets(path) + + +def test_size_limit_truncation() -> None: + huge = {"version": "openlabs.evidence.v1", "steps": [{"stdout": "x" * 500_000}]} + path = _temp_json("truncate-out.json") + write_bounded_json(path, huge, max_bytes=4096) + data = json.loads(path.read_text(encoding="utf-8")) + assert data.get("truncated") is True + assert path.stat().st_size <= 4096 + 64 + + +def test_cleanup_after_failure() -> None: + cleanup_called = {"value": False} + + def fake_cleanup(ctx): # noqa: ANN001 + _ = ctx + cleanup_called["value"] = True + return {"ok": True, "detail": "cleanup ok", "seconds": 0.0, "steps": []} + + def runner(argv: list[str], *, cwd: Path, timeout: float) -> subprocess.CompletedProcess[str]: + _ = cwd, timeout + return _completed(1, "", "fail") + + with mock.patch.object(tier1_mod, "_cleanup", side_effect=fake_cleanup): + code, doc = run_tier1_lifecycle(repo_root=REPO_ROOT, output=None, runner=runner) + assert code != 0 + assert cleanup_called["value"] + assert doc["cleanup"]["ok"] is True + + +def test_missing_executable_structured() -> None: + def runner(argv: list[str], *, cwd: Path, timeout: float) -> subprocess.CompletedProcess[str]: + _ = argv, cwd, timeout + raise FileNotFoundError("openlabs") + + record = run_openlabs_step( + [str(REPO_ROOT / "openlabs"), "--json", "--non-interactive", "lab", "setup", "duck-cross"], + step_name="lab_setup", + cwd=REPO_ROOT, + runner=runner, + ) + assert record["diagnostic_key"] == "environment.docker.client_missing" + + +def test_timeout_structured() -> None: + def runner(argv: list[str], *, cwd: Path, timeout: float) -> subprocess.CompletedProcess[str]: + _ = argv, cwd, timeout + raise subprocess.TimeoutExpired(cmd=argv, timeout=timeout) + + record = run_openlabs_step( + [str(REPO_ROOT / "openlabs"), "--json", "--non-interactive", "lab", "setup", "duck-cross"], + step_name="lab_setup", + cwd=REPO_ROOT, + runner=runner, + ) + assert record["exit_code"] == 124 + + +def test_bound_prove_wrapper() -> None: + raw = _temp_json("raw-prove.json") + dest = _temp_json("bounded-prove.json") + raw.write_text(json.dumps({"lab": "duck-cross", "steps": []}) + "\n", encoding="utf-8") + bound_file(raw, dest) + data = json.loads(dest.read_text(encoding="utf-8")) + assert data["meta"]["redacted"] is True + + +def test_valid_envelope_in_success_step() -> None: + record = run_openlabs_step( + [str(REPO_ROOT / "openlabs"), "--json", "--non-interactive", "lab", "status", "duck-cross"], + step_name="lab_status", + cwd=REPO_ROOT, + runner=lambda argv, cwd, timeout: _completed(0, json.dumps(OK_ENVELOPE), ""), + ) + assert record["ok"] is True + errors = validate_envelope(record["envelope"], path="test.json") + assert not errors + + +def main() -> int: + tests = [ + test_success_sequence_all_steps, + test_command_failure_partial_evidence, + test_malformed_json_from_cli, + test_invalid_envelope_recorded, + test_secret_redaction_in_artifact, + test_size_limit_truncation, + test_cleanup_after_failure, + test_missing_executable_structured, + test_timeout_structured, + test_bound_prove_wrapper, + test_valid_envelope_in_success_step, + ] + failures = 0 + for test in tests: + try: + test() + except Exception as exc: # noqa: BLE001 + failures += 1 + print(f"FAIL {test.__name__}: {exc}", file=sys.stderr) + if failures: + print(f"tier1 lifecycle tests: failed {failures}", file=sys.stderr) + return 1 + print(f"tier1 lifecycle tests: passed {len(tests)} tests") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 56e3ad4060e45dff407b06f4322e3c2983701b82 Mon Sep 17 00:00:00 2001 From: M9nx Date: Fri, 9 Oct 2026 02:43:15 +0300 Subject: [PATCH 4/6] feat(ci): add explicit test_m2 gate and delegate test_contract --- AGENTS.md | 1 + scripts/test_contract.py | 14 +---------- scripts/test_m2.py | 53 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 13 deletions(-) create mode 100644 scripts/test_m2.py diff --git a/AGENTS.md b/AGENTS.md index 1415c444..10b78fb0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,6 +77,7 @@ verifies a player's flag against the stored SHA-256 hash. ```bash python3 scripts/validate.py # structure + metadata +python3 scripts/test_m2.py # non-Docker M2 CLI and lifecycle gates python3 scripts/test_contract.py # contract conformance gate (M1-08) python3 scripts/test_openlabs_cli.py # openlabs CLI core (M2-02) python3 scripts/test_openlabs_issue.py # issue explain and bundle (M2-03) diff --git a/scripts/test_contract.py b/scripts/test_contract.py index 3cbf3b20..f86e929a 100644 --- a/scripts/test_contract.py +++ b/scripts/test_contract.py @@ -11,21 +11,9 @@ SCRIPTS = REPO_ROOT / "scripts" STEPS = ( + "test_m2.py", "test_openlabs_contract.py", "test_contract_schema.py", - "test_diagnostic_registry.py", - "test_cli_contract_fixtures.py", - "test_openlabs_cli.py", - "test_openlabs_issue.py", - "test_openlabs_lab_discovery.py", - "test_openlabs_environment.py", - "test_openlabs_doctor.py", - "test_openlabs_state.py", - "test_openlabs_namespace.py", - "test_openlabs_port.py", - "test_openlabs_lab_setup.py", - "test_openlabs_lifecycle.py", - "test_openlabs_reset_safety.py", ) diff --git a/scripts/test_m2.py b/scripts/test_m2.py new file mode 100644 index 00000000..a80e6ca8 --- /dev/null +++ b/scripts/test_m2.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Local entry point for non-Docker M2 CLI and lifecycle gates.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +SCRIPTS = REPO_ROOT / "scripts" + +STEPS = ( + "test_run_m2_tier1_lifecycle.py", + "test_openlabs_cli.py", + "test_openlabs_issue.py", + "test_openlabs_lab_discovery.py", + "test_openlabs_environment.py", + "test_openlabs_doctor.py", + "test_openlabs_state.py", + "test_openlabs_namespace.py", + "test_openlabs_port.py", + "test_openlabs_lab_setup.py", + "test_openlabs_lifecycle.py", + "test_openlabs_reset_safety.py", + "test_openlabs_verify.py", + "test_cli_contract_fixtures.py", + "test_diagnostic_registry.py", +) + + +def run_step(name: str) -> int: + result = subprocess.run( + [sys.executable, str(SCRIPTS / name)], + cwd=REPO_ROOT, + ) + if result.returncode != 0: + print(f"m2 gate: {name} failed", file=sys.stderr) + return result.returncode + return 0 + + +def main() -> int: + for name in STEPS: + code = run_step(name) + if code != 0: + return code + print(f"m2 gate: passed {len(STEPS)} suites") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 057ba8c23fbcda389e328ef5248778e5da853201 Mon Sep 17 00:00:00 2001 From: M9nx Date: Fri, 9 Oct 2026 02:43:15 +0300 Subject: [PATCH 5/6] ci: dual tier1 proof, bounded artifacts, and expanded routing --- .github/workflows/labs.yml | 66 +++++++++++++++++++++++-- scripts/fixtures/ci_routing_matrix.json | 45 +++++++++++++++++ scripts/test_ci_routing_matrix.py | 22 +++++++++ 3 files changed, 129 insertions(+), 4 deletions(-) diff --git a/.github/workflows/labs.yml b/.github/workflows/labs.yml index a47791ad..771c0b2f 100644 --- a/.github/workflows/labs.yml +++ b/.github/workflows/labs.yml @@ -78,6 +78,13 @@ jobs: - 'scripts/test_openlabs_lab_setup.py' - 'scripts/test_openlabs_lifecycle.py' - 'scripts/test_openlabs_reset_safety.py' + - 'scripts/test_openlabs_verify.py' + - 'scripts/test_m2.py' + - 'scripts/test_run_m2_tier1_lifecycle.py' + - 'scripts/evidence_artifact.py' + - 'scripts/run_m2_tier1_lifecycle.py' + - 'scripts/diagnostic_registry.py' + - 'openlabs' - 'contracts/cli-v1.md' - 'contracts/**' - 'scripts/test_validate*.py' @@ -97,8 +104,12 @@ jobs: - 'scripts/check.py' - '.gitleaks.toml' reference: - - 'labs/web/duck-cross/**' + - 'openlabs' + - 'scripts/openlabs_cli/**' + - 'scripts/run_m2_tier1_lifecycle.py' - 'scripts/prove_reference_lab.py' + - 'scripts/diagnostic_registry.py' + - 'labs/web/duck-cross/**' content: - 'content/**' - 'contentbit.config.ts' @@ -227,7 +238,7 @@ jobs: needs: detect-changes if: needs.detect-changes.outputs.run_reference == 'true' runs-on: ubuntu-latest - timeout-minutes: 12 + timeout-minutes: 25 steps: - uses: actions/checkout@v4 @@ -235,14 +246,61 @@ jobs: with: python-version: "3.12" - - name: Prove reference lab lifecycle - run: python3 scripts/prove_reference_lab.py --json reference-lab-evidence.json + - name: Docker Compose sanity + run: docker compose version + + - name: Prove reference lab lifecycle (legacy) + id: prove_legacy + continue-on-error: true + run: python3 scripts/prove_reference_lab.py --json reference-lab-evidence-raw.json + + - name: Bound legacy reference evidence + if: always() + run: | + if [ ! -f reference-lab-evidence-raw.json ]; then + echo '{"lab":"duck-cross","steps":[],"detail":"legacy prove produced no raw evidence"}' > reference-lab-evidence-raw.json + fi + python3 scripts/evidence_artifact.py bound reference-lab-evidence-raw.json reference-lab-evidence.json + + - name: Tier 1 openlabs lifecycle + id: tier1_lifecycle + if: always() + run: python3 scripts/run_m2_tier1_lifecycle.py --json m2-tier1-lifecycle-evidence.json + + - name: Scan lifecycle artifacts for plaintext flags + if: always() + run: | + python3 scripts/evidence_artifact.py scan-secrets reference-lab-evidence.json + python3 scripts/evidence_artifact.py scan-secrets m2-tier1-lifecycle-evidence.json + + - name: Aggregate Tier 1 lifecycle results + if: always() + run: | + set -euo pipefail + fail=0 + if [ "${{ steps.prove_legacy.outcome }}" != "success" ]; then + echo "legacy prove_reference_lab failed" >&2 + fail=1 + fi + if [ "${{ steps.tier1_lifecycle.outcome }}" != "success" ]; then + echo "openlabs tier1 lifecycle failed" >&2 + fail=1 + fi + exit "$fail" - uses: actions/upload-artifact@v4 if: always() with: name: reference-lab-evidence path: reference-lab-evidence.json + if-no-files-found: error + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: m2-tier1-lifecycle-evidence + path: m2-tier1-lifecycle-evidence.json + if-no-files-found: error security: name: Security scan diff --git a/scripts/fixtures/ci_routing_matrix.json b/scripts/fixtures/ci_routing_matrix.json index 9b4cf3bd..1d4acf66 100644 --- a/scripts/fixtures/ci_routing_matrix.json +++ b/scripts/fixtures/ci_routing_matrix.json @@ -77,6 +77,51 @@ "run_pdf": false, "run_reference": false }, + { + "change": "openlabs wrapper only", + "changed_files": ["openlabs"], + "run_validate": true, + "run_security": false, + "run_content": false, + "run_pdf": false, + "run_reference": true + }, + { + "change": "scripts/openlabs_cli/lab_verify.py only", + "changed_files": ["scripts/openlabs_cli/lab_verify.py"], + "run_validate": true, + "run_security": false, + "run_content": false, + "run_pdf": false, + "run_reference": true + }, + { + "change": "scripts/diagnostic_registry.py only", + "changed_files": ["scripts/diagnostic_registry.py"], + "run_validate": true, + "run_security": false, + "run_content": false, + "run_pdf": false, + "run_reference": true + }, + { + "change": "scripts/test_m2.py only", + "changed_files": ["scripts/test_m2.py"], + "run_validate": true, + "run_security": false, + "run_content": false, + "run_pdf": false, + "run_reference": false + }, + { + "change": "scripts/run_m2_tier1_lifecycle.py only", + "changed_files": ["scripts/run_m2_tier1_lifecycle.py"], + "run_validate": true, + "run_security": false, + "run_content": false, + "run_pdf": false, + "run_reference": true + }, { "change": "wiki/Home.md only", "changed_files": ["wiki/Home.md"], diff --git a/scripts/test_ci_routing_matrix.py b/scripts/test_ci_routing_matrix.py index 96a40138..5e730687 100644 --- a/scripts/test_ci_routing_matrix.py +++ b/scripts/test_ci_routing_matrix.py @@ -28,11 +28,20 @@ ) CONTRACT_PATHS = ( "scripts/test_contract.py", + "scripts/test_m2.py", "scripts/openlabs_contract.py", "contracts/lab.schema.json", "contracts/diagnostics.json", "contracts/cli-v1.md", ) +TIER1_REFERENCE_PATHS = ( + "openlabs", + "scripts/openlabs_cli/lab_verify.py", + "scripts/run_m2_tier1_lifecycle.py", + "scripts/prove_reference_lab.py", + "scripts/diagnostic_registry.py", + "labs/web/duck-cross/README.md", +) def check_structure(data: dict) -> int: @@ -93,6 +102,18 @@ def check_contract_paths_in_filters(filters: dict) -> int: return failures +def check_tier1_reference_paths(filters: dict) -> int: + from ci_routing import path_matches_pattern + + patterns = filters.get("reference", ()) + failures = 0 + for path in TIER1_REFERENCE_PATHS: + if not any(path_matches_pattern(path, pattern) for pattern in patterns): + failures += 1 + print(f"reference filter missing tier1 path coverage for {path}") + return failures + + def check_permissions() -> int: if labs_workflow_permissions_read_only(): return 0 @@ -109,6 +130,7 @@ def main() -> int: print(exc, file=sys.stderr) return 1 failures += check_contract_paths_in_filters(filters) + failures += check_tier1_reference_paths(filters) failures += check_routing_examples(data, filters) failures += check_permissions() if failures: From c2c595e56d3840fc056cc23e26e42cf77362c688 Mon Sep 17 00:00:00 2001 From: M9nx Date: Fri, 9 Oct 2026 02:43:15 +0300 Subject: [PATCH 6/6] docs(ci): regenerate M0 baseline evidence and update Baseline-CI --- scripts/baseline_report.py | 5 ++++- wiki/Baseline-CI.md | 5 +++++ wiki/M0-Baseline-Evidence.md | 7 +++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/baseline_report.py b/scripts/baseline_report.py index aaaf2736..fd7c5eb0 100755 --- a/scripts/baseline_report.py +++ b/scripts/baseline_report.py @@ -162,6 +162,7 @@ def local_ci_commands() -> list[dict]: "ci_job": "Validate labs", "commands": [ "python3 scripts/validate.py", + "python3 scripts/test_m2.py", "python3 scripts/test_contract.py", "python3 scripts/test_validate_status.py", "python3 scripts/test_status_aware_inventory.py", @@ -179,7 +180,9 @@ def local_ci_commands() -> list[dict]: { "ci_job": "Prove duck-cross L0-L6", "commands": [ - "python3 scripts/prove_reference_lab.py --json reference-lab-evidence.json" + "python3 scripts/prove_reference_lab.py --json reference-lab-evidence-raw.json", + "python3 scripts/evidence_artifact.py bound reference-lab-evidence-raw.json reference-lab-evidence.json", + "python3 scripts/run_m2_tier1_lifecycle.py --json m2-tier1-lifecycle-evidence.json", ], }, { diff --git a/wiki/Baseline-CI.md b/wiki/Baseline-CI.md index 29515cd2..3c79b455 100644 --- a/wiki/Baseline-CI.md +++ b/wiki/Baseline-CI.md @@ -64,11 +64,16 @@ provides to Actions on PRs. ## Evidence commands ```bash +python3 scripts/test_m2.py python3 scripts/test_ci_routing_matrix.py bash scripts/lint_workflows.sh python3 scripts/baseline_report.py --check --wiki wiki/M0-Baseline-Evidence.md ``` +The **Prove duck-cross L0-L6** job uploads legacy prove evidence and a Tier 1 +`openlabs` lifecycle JSON. Both artifacts are redacted and size-bounded before +upload. + For the full M0 completion record and local equivalents to every CI job, see [M0 baseline evidence](M0-Baseline-Evidence) ([#72](https://github.com/duckurity/openlabs/issues/72)). Branch protection verification is manual; use [Branch protection record](Branch-Protection-Record). diff --git a/wiki/M0-Baseline-Evidence.md b/wiki/M0-Baseline-Evidence.md index b4ba06d3..e3235861 100644 --- a/wiki/M0-Baseline-Evidence.md +++ b/wiki/M0-Baseline-Evidence.md @@ -2,7 +2,7 @@ Auditable M0 completion record for [#72](https://github.com/duckurity/openlabs/issues/72) (M0-09). Use it before enabling branch protection on `main`. Depends on [#71](https://github.com/duckurity/openlabs/issues/71) (**CI required**). -Reviewed on `2026-09-29`. Regenerate with `python3 scripts/baseline_report.py --write-wiki wiki/M0-Baseline-Evidence.md`. Write `--json` when you need the reviewed git commit and full inventory snapshot. +Reviewed on `2026-10-08`. Regenerate with `python3 scripts/baseline_report.py --write-wiki wiki/M0-Baseline-Evidence.md`. Write `--json` when you need the reviewed git commit and full inventory snapshot. ## Catalog manifest @@ -93,6 +93,7 @@ Run these from a clean checkout when reproducing **CI required** scope on `main` ```bash python3 scripts/validate.py +python3 scripts/test_m2.py python3 scripts/test_contract.py python3 scripts/test_validate_status.py python3 scripts/test_status_aware_inventory.py @@ -112,7 +113,9 @@ python3 scripts/validate.py --compose ### Prove duck-cross L0-L6 ```bash -python3 scripts/prove_reference_lab.py --json reference-lab-evidence.json +python3 scripts/prove_reference_lab.py --json reference-lab-evidence-raw.json +python3 scripts/evidence_artifact.py bound reference-lab-evidence-raw.json reference-lab-evidence.json +python3 scripts/run_m2_tier1_lifecycle.py --json m2-tier1-lifecycle-evidence.json ``` ### Security scan