qa(phase-3): RRI cadence + latency gates, observability (trends/reconcile), split-RRI orchestration, visual regression

.github/workflows/ci.yml

@@ -100,7 +100,10 @@ jobs:
             ../../qa/test_root_cause_analyzer.py \
             ../../qa/test_evidence_audit.py \
             ../../qa/test_triage_failure.py \
-            ../../qa/test_behavioral_gate_corpus.py
+            ../../qa/test_behavioral_gate_corpus.py \
+            ../../qa/test_deterministic_rri_gate.py \
+            ../../qa/test_orchestrate_split_rri.py \
+            ../../qa/test_visual_regression_check.py
 
   server-contracts:
     # Phase-1 (DETERMINISTIC-2): deterministic cross-service MCP contract tests

.github/workflows/release-readiness.yml

@@ -0,0 +1,124 @@
+name: Release Readiness (deterministic, advisory)
+
+# The full Release Readiness Index (RRI) needs a live five-persona sweep + the Mac built-app
+# handoff, which CI cannot mint. But the DETERMINISTIC subset of the release gates — native
+# built-app transition, ui-audit, image-render, palette-live, and the additive per-beat latency
+# budget — needs NO live LLM/persona evidence. This cadence runs
+# `qa/release_readiness.py --deterministic-only` as an EARLY advisory answer to "do the
+# deterministic release gates hold?" — the LLM/persona gates are reported SKIPPED (never FAILED).
+#
+# ADVISORY, NEVER BLOCKING. `continue-on-error` keeps the job from ever going red / blocking a
+# merge; a below-bar deterministic result ANNOTATES (::warning) and uploads RRI-deterministic.json
+# as an artifact for triage. This is a signal, not a gate.
+#
+# Gateway-free / null-backend: no live model, no Eva, no gateway, no global mcp config. It builds
+# a small deterministic fixture in a TEMP dir and points the reader at it, then writes the rollup
+# to the WORKSPACE artifact path — NEVER the committed qa/RRI.json / qa/scores.db / ledger /
+# transcripts. The reader is a pure on-disk reader (the engine stays the sole state writer).
+
+on:
+  # Manual dispatch (the agent / a maintainer can ask "does the deterministic subset hold?").
+  workflow_dispatch:
+    inputs:
+      build_sha:
+        description: "Build SHA to stamp the deterministic rollup with (defaults to the checked-out HEAD)."
+        required: false
+        default: ""
+  # Standing cadence. 23:41 UTC daily, off the top of the hour to dodge the cron thundering herd.
+  schedule:
+    - cron: "41 23 * * *"
+
+# One advisory run at a time; a newer trigger supersedes an in-flight run.
+concurrency:
+  group: release-readiness-deterministic
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  deterministic-rri:
+    runs-on: ubuntu-latest
+    # ADVISORY: never blocks. A below-bar deterministic subset (or a harness hiccup) annotates
+    # and uploads the rollup; it does not fail the workflow.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Run release_readiness.py --deterministic-only (advisory)
+        id: rri
+        # Builds a deterministic five-persona fixture in a TEMP dir (clean part_a PASS, 200
+        # image traffic, healthy under-budget latency) so the deterministic gate machinery runs
+        # end-to-end WITHOUT a live model. The rollup JSON is written to the workspace artifact
+        # path. We never let a non-zero exit fail the step — this cadence ALERTS, never blocks.
+        run: |
+          set -uo pipefail
+          BUILD_SHA="${{ github.event.inputs.build_sha }}"
+          [ -n "$BUILD_SHA" ] || BUILD_SHA="$(git rev-parse --short=12 HEAD)"
+          echo "build_sha=$BUILD_SHA"
+
+          FIX="$(mktemp -d)"
+          ART="${GITHUB_WORKSPACE}/RRI-deterministic.json"
+          echo "ART=$ART" >> "$GITHUB_ENV"
+
+          # Deterministic evidence inputs (no live model / persona involved).
+          printf '{"overall": 5}\n' > "$FIX/story.json"
+          printf '{"overall": 5}\n' > "$FIX/mech.json"
+          printf 'GREEN\n'          > "$FIX/behavioral.txt"
+          printf 'PASS\n'           > "$FIX/audit.log"
+          printf '{"can_act": true}\n' > "$FIX/session_surface.final.json"
+
+          RUNS=""
+          for persona in newbie veteran adversarial narrative optimizer; do
+            d="$FIX/gate-$persona"
+            mkdir -p "$d/player"
+            printf '{"run":"gate-%s","persona":"%s","completed_intro_flow":true,"persona_satisfaction":9,"gave_up":false,"bug_reports_critical":0,"console_errors":0,"image_404s":0}\n' \
+              "$persona" "$persona" > "$d/score.json"
+            printf '{"build_sha":"%s","part_a":{"result":"PASS"},"part_b":{"persona_loop":"PASS","score_pass":true}}\n' \
+              "$BUILD_SHA" > "$d/run.json"
+            printf '{"url":"http://127.0.0.1/image?scope=%s","status":200}\n' "$persona" > "$d/player/network.ndjson"
+            # Healthy under-budget latency sidecar (the additive latency gate path exercises).
+            printf '{"s_per_beat":78.2,"coldopen_s":157.0,"turns_per_beat":4.4}\n' > "$d/latency.json"
+            RUNS="${RUNS:+$RUNS,}$d"
+          done
+
+          # --deterministic-only: evaluates ONLY the gates that need no live LLM/persona evidence
+          # and marks the LLM/persona gates SKIPPED. Pure on-disk reader; --out is the workspace
+          # artifact, never the committed qa/RRI.json.
+          uv run --directory servers/engine --no-project python "${GITHUB_WORKSPACE}/qa/release_readiness.py" \
+            --runs "$RUNS" \
+            --expected-personas "newbie,veteran,adversarial,narrative,optimizer" \
+            --story "$FIX/story.json" --mech "$FIX/mech.json" \
+            --behavioral GREEN --behavioral-path "$FIX/behavioral.txt" \
+            --ui-audit PASS --ui-audit-log "$FIX/audit.log" \
+            --palette-live true --palette-source "$FIX/session_surface.final.json" \
+            --build-sha "$BUILD_SHA" \
+            --deterministic-only \
+            --out "$ART" | tee "$FIX/rri.log" || true
+
+          rm -rf "$FIX"/gate-* "$FIX"/*.json "$FIX"/*.txt "$FIX"/*.log "$FIX" 2>/dev/null || true
+
+          if [ ! -s "$ART" ]; then
+            echo "::warning title=Release Readiness::deterministic rollup produced no RRI-deterministic.json — see logs (advisory)."
+            exit 0
+          fi
+
+          dpass="$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get("deterministic_pass"))' "$ART" 2>/dev/null || echo unknown)"
+          dfail="$(python3 -c 'import json,sys;print(",".join(json.load(open(sys.argv[1])).get("deterministic_failed_gates") or []) or "none")' "$ART" 2>/dev/null || echo unknown)"
+          if [ "$dpass" = "True" ]; then
+            echo "::notice title=Release Readiness::Deterministic release gates HOLD (deterministic_pass=true). LLM/persona gates skipped (advisory)."
+          else
+            echo "::warning title=Release Readiness::Deterministic release gates DID NOT all hold (deterministic_pass=$dpass; failed: $dfail). Advisory — not blocking. See RRI-deterministic.json."
+          fi
+
+      - name: Upload RRI-deterministic.json
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: rri-deterministic-${{ github.run_id }}
+          path: RRI-deterministic.json
+          if-no-files-found: ignore
+          retention-days: 14

qa/latency_baseline.json

@@ -0,0 +1,11 @@
+{
+  "schema": "worldos.latency-baseline.v1",
+  "_comment": "ADDITIVE per-beat latency BUDGET for the release-readiness latency gates (Phase-3). These budgets gate ONLY when a run actually carries latency evidence AND that evidence exceeds the budget; when latency data is ABSENT the gate is a documented EVIDENCE-GAP/skip, never a new false fail, so every existing RRI result is unchanged. Budgets are the healthy ledger figures (qa/scores_ledger.md: duo-baseline s/beat ~78.2, cold-open ~157-161) with headroom so normal scorer/host variance does not trip the gate. s/beat is the MEAN GENERATION seconds over CONTINUING (routine) beats (cold open excluded); coldopen_s is the one-time world-build first beat. Both are sourced from the same on-disk artifacts release_readiness already reads (a run's latency.json sidecar, or a latency block in run.json / score.json) via qa/latency_rollup.py's columns.",
+  "s_per_beat_budget": 120.0,
+  "coldopen_s_budget": 240.0,
+  "_baseline_evidence": {
+    "s_per_beat_healthy": 78.2,
+    "coldopen_s_healthy": 157.0,
+    "source": "qa/scores_ledger.md duo-baseline-rc1b (s/beat 78.2, cold-open 161) and the GUI VM sweeps (cold-open 156.6); budgets add headroom above the healthy figure so routine variance does not false-fail."
+  }
+}