feat(harness): close ten harness gaps found against OpenCode/Hermes, and measure them

.github/workflows/ci-desktop.yml

@@ -23,6 +23,8 @@ on:
       - "tools/cli/package.json"
       - "tools/cli-testing/package.json"
       - "tools/cli-testing/package-lock.json"
+      - "tools/cli-testing/harness/**"
+      - "tools/desktop-journeys/**"
       - "tools/desktop-runtime/manifest.json"
       - "package.json"
       - "package-lock.json"
@@ -43,6 +45,8 @@ on:
       - "tools/cli/package.json"
       - "tools/cli-testing/package.json"
       - "tools/cli-testing/package-lock.json"
+      - "tools/cli-testing/harness/**"
+      - "tools/desktop-journeys/**"
       - "tools/desktop-runtime/manifest.json"
       - "package.json"
       - "package-lock.json"
@@ -148,3 +152,164 @@ jobs:
 
       - name: Test the workspace-fs addon
         run: npm run test:native --workspace @0x-copilot/desktop
+
+  # ── ONE real user journey against the BUILT app, on every PR ────────────────
+  #
+  # tools/desktop-journeys/ drives the real packaged app — supervised Electron +
+  # embedded PostgreSQL + all three Python services — through the Playwright
+  # control server, clicking real testIds. It has repeatedly caught
+  # "unit-tests-pass-but-the-real-app-is-broken" regressions, and until this job
+  # NOTHING ran it on a PR: `grep -rln desktop-journeys .github/workflows/`
+  # returned zero files. desktop-supervised-boot-drill.yml is the nearest thing
+  # and it is schedule + dispatch + tools/desktop-runtime paths only, driving the
+  # stack as a STAGING process rather than through the shipped app.
+  #
+  # Four constraints shape this job. Each is load-bearing; none is style.
+  #
+  # 1. EXACTLY ONE PHASE, pinned by JOURNEY_PHASES below. Since e8622a1d the
+  #    phases in a journey file share ONE boot and consume each other's state on
+  #    purpose, so running the whole file per-PR would inherit stale route / rail
+  #    / env / run-history and fail in a LATER phase with a symptom-shaped
+  #    message that reads exactly like a product bug. FR-0 is first, so it
+  #    consumes nothing.
+  #
+  # 2. NO SECRETS, NO THIRD-PARTY CALL. FR-0 is the only phase that needs no
+  #    provider key and drives no model: it asserts the keyless BYOK gate and
+  #    that the catalog, read through the app, reports nothing configured.
+  #    NOTE ON THE FAKE MODEL: RUNTIME_FAKE_MODEL=1 is what the boot drill uses,
+  #    and it is deliberately unreachable from here. `buildServiceEnv` starts
+  #    from {} and copies only ENV_PASSTHROUGH_ALLOWLIST
+  #    (apps/desktop/main/services/service-env.ts), which does not contain it —
+  #    that is the fail-closed guarantee stated in
+  #    agent_runtime/execution/fake_model.py ("the shipped desktop never sets
+  #    it"). Setting it in this job's env would be a no-op that reads like
+  #    coverage, so it is not set. The hermetic run->stream tail stays where it
+  #    can actually run: desktop-supervised-boot-drill.yml.
+  #
+  # 3. STAGE FRESH. <COPILOT_HOME>/runtime/**/services/** is a COPY of services/*
+  #    taken at stage time and nothing re-syncs it, so a cached stage would test
+  #    code the PR deleted. Only the DOWNLOADS (CPython + PostgreSQL tarballs)
+  #    are cached; the stage itself is rebuilt from this commit's tree.
+  #
+  # 4. NOT A REQUIRED CHECK. A new e2e gate needs a few runs of observed
+  #    stability before anything is allowed to block on it. Requiring it also
+  #    means making it unconditional (see the ci-gates.yml header) — this job is
+  #    path-filtered and macOS-billed, so that is a separate, deliberate step.
+  desktop-journey-first-run:
+    name: desktop journey (first_run FR-0)
+    # Apple Silicon: the staged runtime manifest ships darwin/win32 only, and
+    # this is the arch whose bundle the runner can both download and execute.
+    runs-on: macos-14
+    timeout-minutes: 45
+    env:
+      # THE ONE-PHASE CONSTRAINT, in one place. _lib.py's JourneyPlan runs only
+      # the phases named here and drops the rest; naming an id the journey does
+      # not declare exits 1 rather than passing an empty run, so renaming FR-0
+      # cannot leave this job green over nothing.
+      JOURNEY_PHASES: FR-0
+      # Port 8790 is _lib.py's default and collides with a driver left over from
+      # a previous run. A hosted runner is ephemeral so nothing is left over, and
+      # `concurrency:` above cancels a same-ref run before this one starts —
+      # a distinct port is belt-and-braces, chosen rather than derived so the
+      # value in a failing log is the value written here.
+      CTL_PORT: "8791"
+      # A first boot on a fresh profile runs initdb + migrations + three
+      # uvicorns. Hosted macOS is slower than a laptop at all three, and this
+      # number is measured rather than guessed: on a warm M-series laptop, with
+      # the runtime already staged, launch → the sign-in gate took 110s (94s of
+      # it after the window existed). 420 leaves a wide margin over that.
+      #
+      # It governs BOTH waits, which is the whole reason the job works:
+      #   * _lib.py `_probe()` — how long until the driver's control server
+      #     answers, i.e. until the first WINDOW exists; and
+      #   * `DriverSession.wait_for_app_ready` — how long the app may keep
+      #     showing `boot-gate` (BootProgress.tsx) after that.
+      # Those are different moments. The window appears in ~9s; the services
+      # behind it take ~100s more. Before the second wait existed, every journey
+      # went straight to `wait_for(..., timeout_s=60)` and failed with
+      # "sign-in gate never appeared" — a box timeout wearing a product
+      # failure's clothes, and a red gate on this job's very first run.
+      BOOT_TIMEOUT_S: "420"
+      DRIVER_LAUNCH_TIMEOUT_MS: "300000"
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "22"
+          cache: npm
+
+      # _lib.py needs 3.11+ (StrEnum). Pinned rather than trusting whatever
+      # `python3` the image happens to ship, because that failure is an import
+      # error minutes into a 45-minute job.
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.13"
+
+      # DOWNLOAD cache only (CPython + PostgreSQL tarballs), keyed on the pinned
+      # manifest so a re-pin busts it. The staged tree itself is never cached —
+      # see constraint 3 above. Same cache the boot drill uses.
+      - name: Cache desktop runtime downloads
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/enterprise-desktop-runtime
+          key: desktop-runtime-${{ runner.os }}-arm64-${{ hashFiles('tools/desktop-runtime/manifest.json') }}
+          restore-keys: |
+            desktop-runtime-${{ runner.os }}-arm64-
+
+      # Installs playwright (driver.mjs launches Electron through it) and the
+      # electron binary the driver resolves from this checkout's node_modules.
+      - name: Install Node dependencies
+        run: npm ci --no-audit --no-fund
+
+      # Fresh stage from THIS commit's services/*. Ad-hoc code-signing so the
+      # freshly extracted mach-o binaries execute without a Developer ID.
+      - name: Stage self-contained runtime (darwin-arm64, ad-hoc signed)
+        run: node tools/desktop-runtime/stage.mjs --platform darwin --arch arm64 --adhoc-sign
+
+      # The driver launches apps/desktop/out/**, not the TypeScript sources.
+      - name: Build the desktop bundle
+        run: npm run build --workspace @0x-copilot/desktop
+
+      # THE JOURNEY. Exit 0 only if FR-0 ran and passed: a skip (3) or a blocked
+      # (2) is non-zero by design, so an absent stage or a vanished phase id
+      # cannot be mistaken for success.
+      - name: One live journey phase against the built app
+        env:
+          # The `source` target's default is this path, but naming it is what
+          # keeps the preflight and the launch pointed at the stage built above.
+          COPILOT_HOME: ${{ github.workspace }}/apps/desktop/resources
+        run: python3 tools/desktop-journeys/first_run.py
+
+      # The four supervised services write their logs under the app's userData
+      # dir, which is OUTSIDE the workspace. Copy them next to the screenshots so
+      # one in-workspace path carries every piece of evidence.
+      #
+      # Done in the shell on purpose. An upload `path:` beginning with `~` is
+      # expanded by @actions/glob, not by bash, and `if-no-files-found: warn`
+      # downgrades a miss to a warning — so getting it wrong loses the service
+      # logs silently, in precisely the red run that needed them. `$HOME` here is
+      # expanded by the shell, which is verifiable. No `${{ }}` in the script
+      # body: a substituted value is a script-injection vector (ci-gates header).
+      - name: Collect supervised service logs
+        if: failure()
+        run: |
+          for logs in "$HOME/Library/Application Support/0xCopilot"/journey-first-run-*/logs; do
+            [ -d "$logs" ] || continue
+            dest="tools/desktop-journeys/runs/services-$(basename "$(dirname "$logs")")"
+            mkdir -p "$dest"
+            cp "$logs"/*.log "$dest"/ 2>/dev/null || true
+          done
+          find tools/desktop-journeys/runs -type f | sed 's/^/  evidence: /'
+
+      # Screenshots, driver.log (Electron main), and the supervised services'
+      # logs. Without these a red run on a runner nobody can ssh into is
+      # unreadable.
+      - name: Upload journey evidence
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: desktop-journey-first-run-evidence
+          path: tools/desktop-journeys/runs/**
+          if-no-files-found: warn
+          retention-days: 7

.github/workflows/ci-gates.yml

@@ -64,8 +64,10 @@ jobs:
         with:
           python-version: "3.13"
 
+      # pyyaml: test_desktop_journey_ci.py parses .github/workflows/ci-desktop.yml
+      # to check the per-PR desktop-journey job against the journey it pins.
       - name: Install pytest
-        run: python -m pip install --upgrade pip pytest
+        run: python -m pip install --upgrade pip pytest pyyaml
 
       # Each gate is tested before it is trusted: a gate whose own tests fail is
       # not evidence about the repository, it is a broken instrument. Every gate
@@ -84,8 +86,11 @@ jobs:
             tools/test_check_llm_provider_imports.py \
             tools/test_check_migration_manifest.py \
             tools/test_check_reader_methods.py \
+            tools/test_check_dark_wiring.py \
             tools/test_apply_branch_protection.py \
-            tools/test_cli_release.py
+            tools/test_cli_release.py \
+            tools/test_desktop_journey_ci.py \
+            tools/test_harness_bench_heavy_tasks.py
 
       - name: Dark capabilities
         run: python tools/check_dark_capabilities.py
@@ -101,6 +106,17 @@ jobs:
       - name: Route reachability
         run: python tools/check_route_reachability.py
 
+      # The sub-module sibling of the two gates above. Those catch a whole
+      # module nobody imports and a whole capability no test turns on; this one
+      # catches the same defect one level down, where the population actually
+      # lives -- a symbol only a test reaches, a capability only one store
+      # backend can serve, a validated field spent on an f-string in a prompt.
+      # 306 such symbols already exist and are recorded with a reason each in
+      # tools/dark_wiring_baseline.txt; this fails on the 307th, and on any
+      # baseline line that has gone stale.
+      - name: Dark wiring
+        run: python tools/check_dark_wiring.py
+
       - name: CSS selector shadowing
         run: python tools/check_css_selector_shadowing.py
 

.prettierignore

@@ -13,3 +13,10 @@ tools/design-parity/surfaces/generative-surfaces-v3/design/copilot.css
 # their emission under runner control avoids a second Markdown writer.
 tools/design-parity/surfaces/chat-tool-call-shell/out/*.md
 tools/design-parity/surfaces/generative-surfaces-v3/out/*.md
+
+# Benchmark arm reports are machine-written by tools/harness-bench/rescore.py
+# with `json.dumps(indent=2)`, which always expands arrays. Prettier collapses
+# short ones, so every rescore would otherwise land a whitespace-only diff over
+# real measurements — and re-formatting evidence is how evidence stops being
+# byte-comparable across arms.
+tools/harness-bench/runs/*.json

apps/desktop/main/services/service-env.test.ts

@@ -170,6 +170,33 @@ describe("buildServiceEnv(backend)", () => {
     expect(env.OTEL_SDK_DISABLED).toBe("true");
   });
 
+  it("forwards a COPILOT_HP__ hyperparameter override to the service", () => {
+    // The step ceiling was configurable in settings.py and unreachable in the
+    // packaged app: the allowlist is exact-name and this family is generated.
+    // A live journey phase could not provoke the ceiling for exactly this
+    // reason, while every unit test built settings directly and never crossed
+    // this boundary.
+    const env = buildServiceEnv(
+      "ai-backend",
+      inputs({ COPILOT_HP__EXECUTION__RECURSION_LIMIT: "40" }),
+    );
+    expect(env.COPILOT_HP__EXECUTION__RECURSION_LIMIT).toBe("40");
+  });
+
+  it("forwards only the COPILOT_HP__ family, not arbitrary host env", () => {
+    const env = buildServiceEnv(
+      "ai-backend",
+      inputs({
+        COPILOT_HP__EXECUTION__RUN_DEADLINE_SECONDS: "900",
+        AWS_SECRET_ACCESS_KEY: "must-not-cross",
+        COPILOT_NOT_HP: "must-not-cross",
+      }),
+    );
+    expect(env.COPILOT_HP__EXECUTION__RUN_DEADLINE_SECONDS).toBe("900");
+    expect(env.AWS_SECRET_ACCESS_KEY).toBeUndefined();
+    expect(env.COPILOT_NOT_HP).toBeUndefined();
+  });
+
   it("passes GOOGLE_OAUTH_CLIENT_ID through when set", () => {
     const env = buildServiceEnv(
       "backend",

apps/desktop/main/services/service-env.ts

@@ -14,6 +14,15 @@ import type { SupervisedServiceName } from "./runtime-paths";
 // never the raw desktop process env). PATH/HOME/etc. are baseline process
 // plumbing; the Google OAuth vars are the contract's named passthroughs;
 // the provider keys are dev conveniences that BYOK supersedes in-product.
+/**
+ * Env families forwarded by PREFIX rather than by exact name.
+ *
+ * Deliberately narrow: a prefix widens what crosses into a supervised service,
+ * so it is justified only where the set of names is generated rather than
+ * enumerable. Today that is exactly one family — the hyperparameter overrides.
+ */
+export const ENV_PASSTHROUGH_PREFIXES: readonly string[] = ["COPILOT_HP__"];
+
 export const ENV_PASSTHROUGH_ALLOWLIST: readonly string[] = [
   // Baseline process plumbing.
   "PATH",
@@ -362,6 +371,24 @@ export function buildServiceEnv(
       env[key] = value;
     }
   }
+  // Hyperparameter overrides are a PREFIX FAMILY, not a fixed set of names:
+  // `COPILOT_HP__EXECUTION__RECURSION_LIMIT` and its siblings are derived from
+  // the hyperparameter document, so an exact-name allowlist can never stay
+  // complete as that document grows. `agent_runtime/settings.py:311` calls this
+  // form "the only surface" for setting them — and nothing forwarded it, so a
+  // supervised service ALWAYS took the built-in default and no operator could
+  // change one. The unit suite could not see it: every hyperparameter test
+  // constructs settings directly and never crosses this boundary. Caught by the
+  // RL-2 live journey phase, which could not provoke the step ceiling it
+  // asserts precisely because the ceiling was unsettable in the packaged app.
+  for (const [key, value] of Object.entries(inputs.processEnv)) {
+    if (!ENV_PASSTHROUGH_PREFIXES.some((prefix) => key.startsWith(prefix))) {
+      continue;
+    }
+    if (value !== undefined && value !== "") {
+      env[key] = value;
+    }
+  }
   env.PYTHONPATH = pythonPathValue(inputs.pathDelimiter);
   env.PYTHONUNBUFFERED = "1";
   // Preview connectors OFF by default in the desktop app; the passthrough above