diff --git a/docs/evidence/pr-29-04/README.md b/docs/evidence/pr-29-04/README.md new file mode 100644 index 0000000..735a940 --- /dev/null +++ b/docs/evidence/pr-29-04/README.md @@ -0,0 +1,35 @@ +# PR 29: live classification on Poppy 0.4 + +This refresh targets main at `b2fc42893ce719023681ff0df14a2b485e6d75d3`. The tested implementation and capture source is `3f240b5f43c97a6012b612f1d0f4e2e0cb7e15e9`. + +The runner changes classify missing live capabilities and statically required fixture values as unsupported. Local adapters that claim a scenario still fail. Fixture preflight shares execution's two interpolation passes, checks optional parameters and fixture-known conditions, and defers unknown runtime expressions. The current currency and feedback concurrency tests from main are retained without changes. + +The [previous PR snapshot](https://github.com/adewale/pda/tree/d02855ec490ee0076ecc9a1549dba30004b012a3) preserves the October reports, 0.3 screenshots and their verification records. Those records are historical and are not carried into this focused change. + +## Verification + +- [Harness](interop.txt): 474 tests passed in 34 files. +- [Negative control](preflight-disabled.txt): all four CLI regressions fail when fixture preflight is disabled. The source was restored before the full suite and capture. +- [Full suite](tests-summary.txt): **2,705 passed, one existing skip**, across 134 files. +- Type checking, pinned-spec integrity, documentation links and all 17 tooling tests passed. +- [Validation receipt](validation.json) records source and artifact hashes. + +## Visual evidence + +Reviewer cue: `fixture-live` changes from red **fail** to grey **n/a** for both `CL` and `ML`. `fixture-claimed` stays red **fail** in both scenarios. + +| Before: absent live capabilities fail | After: absent live capabilities are unsupported | +| --- | --- | +| ![Base 0.4 report: both fixture columns fail CL and ML](before.png) | ![Updated 0.4 report: live capabilities are unsupported, while claimed-support controls still fail](after.png) | + +The same synthetic discovery response runs through actual immutable `runCell` and scenario sources. The production renderer and its complete runtime dependency closure are loaded from each revision, checked for equality and hashed. Chromium renders both at 1400 × 1000; external page requests are blocked. Assertions check all four outcomes, that only discovery runs, and the rendered grid counts. [Receipt](visual-receipt.json), [base HTML](before.html), [updated HTML](after.html). + +Reproduce with the pinned dependencies, Node 24 and Chromium: + +```sh +node docs/evidence/pr-29-04/capture.mjs \ + b2fc42893ce719023681ff0df14a2b485e6d75d3 \ + 3f240b5f43c97a6012b612f1d0f4e2e0cb7e15e9 +``` + +These are synthetic classification checks. They do not establish live Company interoperability or an authenticated hosted journey. No public Company was contacted for this refresh, and the registry's permitted live scenarios are unchanged. diff --git a/docs/evidence/pr-29-04/after.html b/docs/evidence/pr-29-04/after.html new file mode 100644 index 0000000..da5fdf9 --- /dev/null +++ b/docs/evidence/pr-29-04/after.html @@ -0,0 +1,85 @@ + + + + +Poppy interop PR29 synthetic capability fixture + +

Poppy interop run PR29 synthetic capability fixture

+

Lane synthetic; no external traffic · Poppy 0.4 · 2 unsupported, 2 fail. +Checks: 0 failures, 0 warnings, 0 tentative (our readings); only failures fail a cell (docs/conformance-levels.md). +Each failed cell is blamed by the first rule that decides it: violation, warning, grid, reading, or unattributed (docs/interop-spec.md, section 9).

+ + +

Implementations

+ + +
namerolecommitreadingsinterop-only relaxations
pdaagent3f240b5f43c97a6012b612f1d0f4e2e0cb7e15e9Synthetic discover response; actual runCell/report code
fixture-livecompanyfixtureNo conversations or MCP
fixture-claimedcompanyfixtureNo conversations or MCP
+
+

CL: Conversation at a live Company: a labelled test message, read, a follow-up, then close

+

Spec 8.2, 8.3, 8.5, 8.12

+ +
agent \ Companyfixture-livefixture-claimed
pdan/afail
+
+
+

ML: MCP tool listing without calling a tool

+

Spec 7

+ +
agent \ Companyfixture-livefixture-claimed
pdan/afail
+
+ +

Cells

+
+CL pda × fixture-live: n/a +

step 1 (expect): the Company doesn't offer what CL tests: ${d.conversations}: expected true, got false

+ + + + +
#actoropmsresult
0agentdiscover0ok
1expectexpect0${d.conversations}: expected true, got false
+

No checks reported anything.

+
+
+CL pda × fixture-claimed: fail +

step 1 (expect): ${d.conversations}: expected true, got false

+ + + + +
#actoropmsresult
0agentdiscover0ok
1expectexpect0${d.conversations}: expected true, got false
+

No checks reported anything.

+
+
+ML pda × fixture-live: n/a +

step 1 (expect): the Company doesn't offer what ML tests: ${d.apis}: expected $contains "mcp", got []

+ + + + +
#actoropmsresult
0agentdiscover0ok
1expectexpect0${d.apis}: expected $contains "mcp", got []
+

No checks reported anything.

+
+
+ML pda × fixture-claimed: fail +

step 1 (expect): ${d.apis}: expected $contains "mcp", got []

+ + + + +
#actoropmsresult
0agentdiscover0ok
1expectexpect0${d.apis}: expected $contains "mcp", got []
+

No checks reported anything.

+
+ diff --git a/docs/evidence/pr-29-04/after.png b/docs/evidence/pr-29-04/after.png new file mode 100644 index 0000000..ac8869d Binary files /dev/null and b/docs/evidence/pr-29-04/after.png differ diff --git a/docs/evidence/pr-29-04/before.html b/docs/evidence/pr-29-04/before.html new file mode 100644 index 0000000..263e917 --- /dev/null +++ b/docs/evidence/pr-29-04/before.html @@ -0,0 +1,85 @@ + + + + +Poppy interop PR29 synthetic capability fixture + +

Poppy interop run PR29 synthetic capability fixture

+

Lane synthetic; no external traffic · Poppy 0.4 · 4 fail. +Checks: 0 failures, 0 warnings, 0 tentative (our readings); only failures fail a cell (docs/conformance-levels.md). +Each failed cell is blamed by the first rule that decides it: violation, warning, grid, reading, or unattributed (docs/interop-spec.md, section 9).

+ + +

Implementations

+ + +
namerolecommitreadingsinterop-only relaxations
pdaagentb2fc42893ce719023681ff0df14a2b485e6d75d3Synthetic discover response; actual runCell/report code
fixture-livecompanyfixtureNo conversations or MCP
fixture-claimedcompanyfixtureNo conversations or MCP
+
+

CL: Conversation at a live Company: a labelled test message, read, a follow-up, then close

+

Spec 8.2, 8.3, 8.5, 8.12

+ +
agent \ Companyfixture-livefixture-claimed
pdafailfail
+
+
+

ML: MCP tool listing without calling a tool

+

Spec 7

+ +
agent \ Companyfixture-livefixture-claimed
pdafailfail
+
+ +

Cells

+
+CL pda × fixture-live: fail +

step 1 (expect): ${d.conversations}: expected true, got false

+ + + + +
#actoropmsresult
0agentdiscover0ok
1expectexpect1${d.conversations}: expected true, got false
+

No checks reported anything.

+
+
+CL pda × fixture-claimed: fail +

step 1 (expect): ${d.conversations}: expected true, got false

+ + + + +
#actoropmsresult
0agentdiscover0ok
1expectexpect0${d.conversations}: expected true, got false
+

No checks reported anything.

+
+
+ML pda × fixture-live: fail +

step 1 (expect): ${d.apis}: expected $contains "mcp", got []

+ + + + +
#actoropmsresult
0agentdiscover0ok
1expectexpect0${d.apis}: expected $contains "mcp", got []
+

No checks reported anything.

+
+
+ML pda × fixture-claimed: fail +

step 1 (expect): ${d.apis}: expected $contains "mcp", got []

+ + + + +
#actoropmsresult
0agentdiscover0ok
1expectexpect0${d.apis}: expected $contains "mcp", got []
+

No checks reported anything.

+
+ diff --git a/docs/evidence/pr-29-04/before.png b/docs/evidence/pr-29-04/before.png new file mode 100644 index 0000000..748c7e4 Binary files /dev/null and b/docs/evidence/pr-29-04/before.png differ diff --git a/docs/evidence/pr-29-04/capture.mjs b/docs/evidence/pr-29-04/capture.mjs new file mode 100644 index 0000000..7382dc8 --- /dev/null +++ b/docs/evidence/pr-29-04/capture.mjs @@ -0,0 +1,95 @@ +/** One-off PR evidence: use immutable runCell sources and the unchanged production report renderer. */ +import assert from "node:assert/strict"; +import { execFileSync, spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import puppeteer from "@cloudflare/puppeteer"; + +const [base, head] = process.argv.slice(2); +assert.ok([base, head].every(sha => /^[a-f0-9]{40}$/.test(sha)), "Supply full base/head SHAs"); +const root = path.resolve(import.meta.dirname, "../../.."); +const output = import.meta.dirname; +const directory = mkdtempSync(path.join(tmpdir(), "pr29-04-classification-")); +const git = (sha, file) => execFileSync("git", ["show", `${sha}:${file}`], { cwd: root }); +const hash = bytes => createHash("sha256").update(bytes).digest("hex"); +const rendererSources = [ + "interop/runner/report.ts", "interop/runner/attribution.ts", "interop/runner/readings.ts", "interop/runner/har.ts", + "src/validator/levels.ts", "src/poppy/protocol-version.ts" +]; +for (const file of rendererSources) assert.deepEqual(git(base, file), git(head, file), `Renderer dependency must be identical: ${file}`); +const receipt = { + base, head, fixture: "Company offers neither conversations nor MCP", networkRequests: 0, + rendererSha256: Object.fromEntries(rendererSources.map(file => [file, hash(git(head, file))])), stages: [] +}; +let child, browser; +try { + child = spawn(process.env.PDA_ACCEPTANCE_CHROMIUM || "/usr/bin/chromium", ["--headless", "--no-sandbox", "--disable-dev-shm-usage", "--disable-background-networking", "--disable-component-update", "--no-first-run", "--remote-debugging-port=0", `--user-data-dir=${path.join(directory, "chrome")}`], { stdio: ["ignore", "ignore", "pipe"] }); + const endpoint = await new Promise((resolve, reject) => { + let stderr = ""; + const timer = setTimeout(() => reject(new Error("Chromium startup timed out")), 10_000); + child.once("error", error => { clearTimeout(timer); reject(error); }); + child.stderr.on("data", chunk => { + stderr = (stderr + String(chunk)).slice(-8192); + const match = /DevTools listening on (ws:\/\/127\.0\.0\.1:[0-9]+\/devtools\/browser\/[a-z0-9-]+)/.exec(stderr); + if (match) { clearTimeout(timer); resolve(match[1]); } + }); + }); + browser = await puppeteer.connect({ browserWSEndpoint: endpoint }); + const page = await browser.newPage(); + await page.setViewport({ width: 1400, height: 1000, deviceScaleFactor: 1 }); + await page.setRequestInterception(true); + page.on("request", request => { void request.abort(); }); + for (const [stage, sha] of [["before", base], ["after", head]]) { + const source = path.join(directory, stage); + for (const file of ["interop/runner/cell.ts", "interop/runner/scenario.ts", ...rendererSources]) { + const target = path.join(source, file); + mkdirSync(path.dirname(target), { recursive: true }); + writeFileSync(target, git(sha, file)); + } + const { runCell } = await import(pathToFileURL(path.join(source, "interop/runner/cell.ts"))); + const { renderReport } = await import(pathToFileURL(path.join(source, "interop/runner/report.ts"))); + const scenarios = ["CL", "ML"].map(id => JSON.parse(git(sha, `interop/scenarios/${id}.json`))); + const cells = []; + for (const scenario of scenarios) for (const live of [true, false]) { + const calls = []; + const driver = { + agent: async op => { + calls.push(op); + assert.equal(op, "discover", "An absent feature must stop before sending or listing"); + return { kind: "ok", result: { poppy_version: "0.4", conversations: false, apis: [] } }; + }, + company: async () => { throw new Error("No Company calls in this fixture"); }, + runner: async () => { throw new Error("No runner calls in this fixture"); } + }; + const run = await runCell(scenario, {}, driver, { live }); + assert.deepEqual(calls, ["discover"]); + assert.equal(run.outcome, stage === "after" && live ? "unsupported" : "fail"); + cells.push({ agent: "pda", company: live ? "fixture-live" : "fixture-claimed", scenario: scenario.id, protocolVersion: "0.4", outcome: run.outcome, reason: run.reason, blame: null, rule: null, steps: run.steps, checks: [], har: null }); + } + const html = renderReport({ + run: "PR29 synthetic capability fixture", lane: "synthetic; no external traffic", spec: "Poppy 0.4", + implementations: [ + { name: "pda", role: "agent", commit: sha, readings: {}, relaxations: ["Synthetic discover response; actual runCell/report code"] }, + ...["fixture-live", "fixture-claimed"].map(name => ({ name, role: "company", commit: "fixture", readings: {}, relaxations: ["No conversations or MCP"] })) + ], scenarios, cells + }); + const htmlFile = path.join(output, `${stage}.html`); + const pngFile = path.join(output, `${stage}.png`); + writeFileSync(htmlFile, html); + await page.setContent(html, { waitUntil: "load", timeout: 10_000 }); + await page.waitForSelector(".grid"); + assert.equal(await page.$$eval(".grid td.fail", cells => cells.length), stage === "before" ? 4 : 2); + assert.equal(await page.$$eval(".grid td.unsupported", cells => cells.length), stage === "before" ? 0 : 2); + await page.evaluate(() => { window.scrollTo(0, 0); }); + writeFileSync(pngFile, await page.screenshot({ type: "png" })); + receipt.stages.push({ stage, source: sha, cells: cells.map(({ company, scenario, outcome }) => ({ company, scenario, outcome })), htmlSha256: hash(Buffer.from(html)) }); + } + writeFileSync(path.join(output, "visual-receipt.json"), `${JSON.stringify(receipt, null, 2)}\n`); +} finally { + await browser?.close().catch(() => undefined); + child?.kill("SIGTERM"); + rmSync(directory, { recursive: true, force: true }); +} diff --git a/docs/evidence/pr-29-04/interop.txt b/docs/evidence/pr-29-04/interop.txt new file mode 100644 index 0000000..6be8096 --- /dev/null +++ b/docs/evidence/pr-29-04/interop.txt @@ -0,0 +1,7 @@ +RUN v4.1.12 /workspace/pda + + + Test Files 34 passed (34) + Tests 474 passed (474) + Start at 23:30:34 + Duration 6.44s (transform 2.36s, setup 0ms, import 5.71s, tests 6.58s, environment 4ms) diff --git a/docs/evidence/pr-29-04/preflight-disabled.txt b/docs/evidence/pr-29-04/preflight-disabled.txt new file mode 100644 index 0000000..8a515da --- /dev/null +++ b/docs/evidence/pr-29-04/preflight-disabled.txt @@ -0,0 +1,13 @@ +Control: disable only fixture preflight in cli.ts; run the four tests matching "before any adapter call". +All four must fail. Source restored before normal verification. + + × rejects a live nested reference before any adapter call 23ms + × rejects a live embedded reference before any adapter call 7ms + × rejects a live optional step before any adapter call 5ms + × rejects a live active condition before any adapter call 5ms +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 4 ⎯⎯⎯⎯⎯⎯⎯ + FAIL |interop| interop/test/runner.test.ts > rejects a live nested reference before any adapter call + FAIL |interop| interop/test/runner.test.ts > rejects a live embedded reference before any adapter call + FAIL |interop| interop/test/runner.test.ts > rejects a live optional step before any adapter call + FAIL |interop| interop/test/runner.test.ts > rejects a live active condition before any adapter call + Tests 4 failed | 37 skipped (41) diff --git a/docs/evidence/pr-29-04/tests-summary.txt b/docs/evidence/pr-29-04/tests-summary.txt new file mode 100644 index 0000000..82dbaa5 --- /dev/null +++ b/docs/evidence/pr-29-04/tests-summary.txt @@ -0,0 +1,11 @@ + Test Files 66 passed (66) + Tests 1704 passed | 1 skipped (1705) + Test Files 1 passed (1) + Tests 3 passed (3) + Test Files 19 passed (19) + Tests 314 passed (314) + Test Files 34 passed (34) + Tests 474 passed (474) + Test Files 14 passed (14) + Tests 210 passed (210) +Total: 2705 passed, 1 skipped, 134 files. diff --git a/docs/evidence/pr-29-04/tooling.txt b/docs/evidence/pr-29-04/tooling.txt new file mode 100644 index 0000000..9868e29 --- /dev/null +++ b/docs/evidence/pr-29-04/tooling.txt @@ -0,0 +1,28 @@ +> pda@0.0.0 test:tooling +> node --test scripts/*.test.mjs + +✔ stable evidence links work without Vitest or test-title discovery (6.259509ms) +✔ missing files and scenarios fail; an unlinked test claim fails (2.616959ms) +✔ references cannot escape the repository (1.074193ms) +✔ a directory is not a stable requirement-evidence file (1.449125ms) +✔ all project guides check local fragments, including duplicate headings and explicit aliases (2.912142ms) +✔ self links and setext headings work, while code and comment examples supply no anchors or links (1.235775ms) +✔ encoded local paths and fragments are decoded separately and cannot escape the repository (2.162477ms) +✔ directory links use their README for fragments and project links ignore external services (1.913002ms) +✔ mixed backtick runs and escaped openers do not hide real links (2.329988ms) +✔ equal delimiter runs still hide examples, including escaped ticks inside a code span (2.265753ms) +✔ unmatched backticks cannot hide links across paragraph or block boundaries (2.987446ms) +✔ HTML aliases inside inline code supply no anchor, while real aliases and code heading text work (0.940141ms) +✔ backticks cannot cross ordered-list items, while a noninterrupting number remains paragraph text (2.206123ms) +✔ public preflight verifies provenance and never follows OAuth or sends a business request (52.874012ms) +✔ public preflight rejects dirty or mismatched deployments before claiming a pass (2.948102ms) +✔ documentation-only PRs omit runtimes; main always runs everything (1.650648ms) +✔ frontend changes retain runtime tests; server, config and unknown inputs retain specialist checks (1.549817ms) +ℹ tests 17 +ℹ suites 0 +ℹ pass 17 +ℹ fail 0 +ℹ cancelled 0 +ℹ skipped 0 +ℹ todo 0 +ℹ duration_ms 143.124456 diff --git a/docs/evidence/pr-29-04/validation.json b/docs/evidence/pr-29-04/validation.json new file mode 100644 index 0000000..dc6a19e --- /dev/null +++ b/docs/evidence/pr-29-04/validation.json @@ -0,0 +1,39 @@ +{ + "base": "b2fc42893ce719023681ff0df14a2b485e6d75d3", + "source": "3f240b5f43c97a6012b612f1d0f4e2e0cb7e15e9", + "previousPrHead": "d02855ec490ee0076ecc9a1549dba30004b012a3", + "protocol": "0.4", + "checks": { + "passed": 2705, + "skipped": 1, + "files": 134, + "harnessPassed": 474, + "toolingPassed": 17, + "preflightDisabledFailures": 4, + "typecheck": true, + "specIntegrity": true + }, + "sourceSha256": { + "interop/runner/cell.ts": "44195d76bf70b83be4ea6b27d40f0fc8d8968598c487b744cfa600cefd74c522", + "interop/runner/cli.ts": "6db94d1aff4a54edad474720f1cba5ee775c07ffa6f4f67002f64f314f63db02", + "interop/runner/scenario.ts": "529a241cdcef3fbb2ffb7c3e16b96bea87be8a66aea360aafe7f1a5d2322013b", + "interop/runner/types.ts": "35dff4d40d9c7c037d1e88adc28561b8d3ee9cb84e6b92c03b0ed18b8897a26a", + "interop/scenarios/C.json": "8405b2596287c7e7a6999e2f936283a356580895a43016be95b8aee7112c5f95", + "interop/scenarios/CL.json": "9200e51584683807cbdfa7cb2610568adc54b2afc7243cbeb77d712a45f7b8d1", + "interop/scenarios/ML.json": "88c1a6afb1d54061e269d1112cdf7f352d9bee5b04ead3981bc6f3c0fe8956d2", + "interop/test/runner.test.ts": "94748757c30a8c6e4824d549ea684bb8623c6482fdaeb27b6b98390bd78b7d52", + "interop/test/scenario.test.ts": "015bdae8fbbfd3a3bfdf629ab272ea7efb78e115086942d0698f8acbf55719df" + }, + "artifactSha256": { + "after.html": "973270946333dfda3d5a4d542fd9a8fd34aec95b99423e57078f8a6c81d30c2b", + "after.png": "abb96b75830ff29c88877d704f70f2ae222189d6e08644062f9c401cbb135ce2", + "before.html": "fb033ca5c7347b02dc988ced1b56d48ca1604b076a7797e781e98664b93d5f19", + "before.png": "689e9306fa89c2765ef8d3e42f63e0f80de166741bf3eaa6dda242e79078da80", + "capture.mjs": "c4bc793e644d7c3f5d5e27cfddaa515aa22364dd01865d190a78352452371ecd", + "interop.txt": "fc33aff86213de6e824d57954a4356c747ce787ee861067dafad3d75f3d6ceda", + "preflight-disabled.txt": "1a54907cd9db3f83e13b7e87e67b0e5fea05c68ba34bd3b0798cf577e610b21d", + "tests-summary.txt": "aea30815a0f81c6c689feab2f254e07940528daf0b356846437188cdd30cefc0", + "tooling.txt": "a644dcdad3a97b7e6d60d96a7c3517c8d3010f5814c359f039c7a55c5e672cd3", + "visual-receipt.json": "fe5d1b1c2370c9b384e1c580cac613652a7869af77d3fae249b2c4e5b9f67acf" + } +} diff --git a/docs/evidence/pr-29-04/visual-receipt.json b/docs/evidence/pr-29-04/visual-receipt.json new file mode 100644 index 0000000..8827227 --- /dev/null +++ b/docs/evidence/pr-29-04/visual-receipt.json @@ -0,0 +1,70 @@ +{ + "base": "b2fc42893ce719023681ff0df14a2b485e6d75d3", + "head": "3f240b5f43c97a6012b612f1d0f4e2e0cb7e15e9", + "fixture": "Company offers neither conversations nor MCP", + "networkRequests": 0, + "rendererSha256": { + "interop/runner/report.ts": "22cef42a6b0dade87ec68e65801d6cf846bb22561562533f9a2766fea1e5c8f2", + "interop/runner/attribution.ts": "698a2ccd3ff6839e86242b8b40f052a5b33427aa8e5d8768e49f5c73add18463", + "interop/runner/readings.ts": "b4ccfbcb8b23e3ca8a701ea026ee6d8f785109e988b5d003bf1a3c41c997670d", + "interop/runner/har.ts": "390051f871f513c51642fe8cf5267d63d23501f89cde5a04c934caadf67a1b02", + "src/validator/levels.ts": "b9c4a2105f4c074d99b7286d36948754bb48c545ee61ef836e2e4356e27c965a", + "src/poppy/protocol-version.ts": "65f5a43f15def3c37ceb30dd00525c1c974983c53ac7fcd5dc3fb6f40649e59b" + }, + "stages": [ + { + "stage": "before", + "source": "b2fc42893ce719023681ff0df14a2b485e6d75d3", + "cells": [ + { + "company": "fixture-live", + "scenario": "CL", + "outcome": "fail" + }, + { + "company": "fixture-claimed", + "scenario": "CL", + "outcome": "fail" + }, + { + "company": "fixture-live", + "scenario": "ML", + "outcome": "fail" + }, + { + "company": "fixture-claimed", + "scenario": "ML", + "outcome": "fail" + } + ], + "htmlSha256": "fb033ca5c7347b02dc988ced1b56d48ca1604b076a7797e781e98664b93d5f19" + }, + { + "stage": "after", + "source": "3f240b5f43c97a6012b612f1d0f4e2e0cb7e15e9", + "cells": [ + { + "company": "fixture-live", + "scenario": "CL", + "outcome": "unsupported" + }, + { + "company": "fixture-claimed", + "scenario": "CL", + "outcome": "fail" + }, + { + "company": "fixture-live", + "scenario": "ML", + "outcome": "unsupported" + }, + { + "company": "fixture-claimed", + "scenario": "ML", + "outcome": "fail" + } + ], + "htmlSha256": "973270946333dfda3d5a4d542fd9a8fd34aec95b99423e57078f8a6c81d30c2b" + } + ] +} diff --git a/docs/interop-spec.md b/docs/interop-spec.md index 33b1ee0..20f0ab4 100644 --- a/docs/interop-spec.md +++ b/docs/interop-spec.md @@ -206,6 +206,7 @@ A scenario is a JSON file in `interop/scenarios/`: - **Adversarial fields:** `"adversary": "company" | "agent" | "network"` marks the side that misbehaves (`network` is the fault-injection proxy). `"faults"` is the proxy's fault list (section 3). `"permitted_by": "N10"` names a draft requirement that permits the attack, so a failure is the spec's, not the honest side's (section 9, rule 0). An `arm` step (an agent or Company op) arms the attack before it is sprung. - A step whose operation the adapter answers 501 ends the cell as `unsupported`, unless the step is `"optional": true`. - `"when": "${name.path}"` runs a step only when the reference names something other than null or false; otherwise the step is recorded as skipped. It is for steps one side's design needs and another's doesn't (approving a held write, section 5.2). +- `"offers": true` on an `expect` step identifies a capability prerequisite: conversations for `C` and `CL`, or MCP for `ML`. A missing capability makes a live cell `unsupported`; a manifest-backed adapter claiming that scenario still fails. - The scenarios are the files in `interop/scenarios/`; [interop-testing.md](history/interop-testing.md#scenarios) lists them by area. The five added last were named there before they were written, so each is defined from the draft section it targets: - `H` (8.7 to 8.9): a conversation, a handoff answered with `conversation_id`, the Company's `staff_join`, then a read showing a Company message whose `author.type` is `human`. - `DC` (8.3, 8.8, 8.10, 8.12): a conversation, then a Direct Conversation under it (`send` with `parent_conversation` and `author.type` `human`), a reply in it, the parent refusing a message with `direct_conversation_open`, the User's words, `close`, and the parent taking a message again. @@ -315,7 +316,7 @@ The runner also writes `index.html`: the grid, one table per scenario with agent ## 10. Lanes - **Local:** every implementation in a container, on the private, internal network, with no way to the internet (section 3, Isolation). The default. -- **Live:** public deployments, signed-out scenarios only (`D`, `S0`, `ML` (MCP listing, no tool call), `CL` where the Company allows signed-out conversations: one labelled test message, a read, a follow-up, then a close, so nothing is left open. `C`, which leaves the conversation open, and `CC`, which needs the Company to close it, run only in the local grid), at most one request a second per Company. The live lane uses real names and public TLS, and only our own agents (`pda`, and the lenient and strict fakes), which need the internet, so the runner keeps it apart from the local lane: it creates no internal network and runs our agents alone, on Docker's default bridge with its control port published, and no third-party image. It first ran on 13 October 2026, against `alpine-retail-live` with its developer's approval. A live Company has no adapter, so it is never reset, its hooks answer 501 (an optional hook step is skipped), and it gets a fixture of one greeting, which says it is a test and needs no action (`LIVE_GREETING` in `interop/runner/cli.ts`). Companies are registry entries with `url` and an expected `protocolVersion`; none is registered since the 0.2 ones were retired (section 14). A discovered version mismatch stops the cell before Session authentication; scenarios are those marked `live` that need no Company hook; nothing is recorded, so only steps and expectations decide a cell. Agents pace themselves from `INTEROP_PACE_MS`. +- **Live:** public deployments, signed-out scenarios only (`D`, `S0`, `ML` (MCP listing, no tool call), `CL` where the Company allows signed-out conversations: one labelled test message, a read, a follow-up, then a close, so nothing is left open. `C`, which leaves the conversation open, and `CC`, which needs the Company to close it, run only in the local grid), at most one request a second per Company. The live lane uses real names and public TLS, and only our own agents (`pda`, and the lenient and strict fakes), which need the internet, so the runner keeps it apart from the local lane: it creates no internal network and runs our agents alone, on Docker's default bridge with its control port published, and no third-party image. It first ran on 13 October 2026, against `alpine-retail-live` with its developer's approval. A live Company has no adapter, so it is never reset, its hooks answer 501 (an optional hook step is skipped), and its registry entry supplies fixture data alongside the labelled test greeting (`LIVE_GREETING` in `interop/runner/cli.ts`). Planning and execution build that fixture with the same helper. A missing statically required fixture value makes a live cell unsupported before any scenario request. Preflight shares execution's two interpolation passes, checks optional-step parameters, and follows fixture-known conditions; unknown runtime conditions and expressions are deferred. A missed `offers` expectation is unsupported too. Companies are registry entries with `url` and an expected `protocolVersion`; current 0.4 entries allow discovery at the real store and discovery plus signed-out Session reads at its sandbox (section 15). A discovered version mismatch stops the cell before Session authentication; scenarios are those marked `live` that need no Company hook; nothing is recorded, so only steps and expectations decide a cell. Agents pace themselves from `INTEROP_PACE_MS`. - **The live lane's identity.** A public Company fetches the agent's `client_id`, so the live lane needs one on the internet. Our deployed Worker serves a separate interop test client at `/poppy/interop/client.json` with its public keys at `/poppy/interop/jwks.json`, only when the deployment sets `INTEROP_LIVE_JWKS`. It has its own key, never the real client's, and no redirect URI. Its private key stays on the runner's machine, in the gitignored `interop/.live-identity.json` that `scripts/interop-live-identity.mjs` writes. When that file exists, the runner passes the `client_id` and the key to our own agents' containers (not one marked `"live": false`) (`INTEROP_LIVE_CLIENT_ID`, and `INTEROP_LIVE_KEY` through the docker CLI's environment, not its arguments), and the agent signs as that client. No other implementation gets them, and the local lane never uses them. Without the file the agent names itself at `pda.agent.interop.test`, which no public Company can fetch: a Company that checks client metadata refuses `S0`, `N` and `CL` with `invalid_client`, and only `D` says anything. The runner says which identity it is using before it starts, then fetches it as a Company would, without following redirects (`checkLiveIdentity` in `interop/runner/expectations.ts`): the `client_id` document must be JSON naming itself, with `private_key_jwt` and a key set on the same origin, and that key set must hold the runner's key, the same `kid` and public point. Anything else stops the run before an agent starts or a request reaches a Company. A `client_id` on another origin than `deployment.local.json`'s `PUBLIC_ORIGIN` only warns, since a static host is a supported setup. - **The live lane never runs adversarial cells.** Adversarial implementations and scenarios with an `adversary` or `faults` list are dropped from the live lane: we never attack a public deployment, whoever's it is. - **Our fakes, but not the sloppy one.** The lenient and strict fake agents send what `pda` sends (the lenient one adds retries naming the same Company), so the live lane runs them, with the same identity. The strict agent matters most there: the live lane records nothing, so its findings at `GET /violations` are the only checks a live cell gets; in the live lane it takes every host but its own for the Company's. The sloppy agent is `"live": false`: its departures are deliberate, and a public deployment hasn't agreed to them. diff --git a/interop/runner/cell.ts b/interop/runner/cell.ts index b6d0c3e..d187550 100644 --- a/interop/runner/cell.ts +++ b/interop/runner/cell.ts @@ -73,7 +73,8 @@ export function withWebPage(fixture: unknown, domain: string): Record { +/** `live`: the Company is a public deployment with no manifest, so an `offers` miss is unsupported (spec §10). */ +export async function runCell(scenario: Scenario, fixture: unknown, driver: Driver, options: { live?: boolean } = {}): Promise { const saved: Saved = { fixture: withScopes(fixture) }; const steps: StepRecord[] = []; const mayRefuse = expectedRefusals(scenario); @@ -88,6 +89,9 @@ export async function runCell(scenario: Scenario, fixture: unknown, driver: Driv if (actor === "expect") { const failures = evaluate(step.expect!, saved); record(failures.length === 0, failures.length ? { error: failures.join("; ") } : {}); + if (failures.length && step.offers && options.live) { + return end("unsupported", `step ${index} (expect): the Company doesn't offer what ${scenario.id} tests: ${failures.join("; ")}`); + } if (failures.length) return end("fail", `step ${index} (expect): ${failures.join("; ")}`); continue; } diff --git a/interop/runner/cli.ts b/interop/runner/cli.ts index 8693185..5567e14 100644 --- a/interop/runner/cli.ts +++ b/interop/runner/cli.ts @@ -28,7 +28,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { parseArgs } from "node:util"; import { agentRecorded, attributeAll, companyRecorded, levelCounts, staleKnown, unexpected, type RecordedViolation } from "./attribution.ts"; -import { runCell, toReply, withWebPage, type Driver, type Reply, type RunnerReply } from "./cell.ts"; +import { runCell, toReply, withScopes, withWebPage, type Driver, type Reply, type RunnerReply } from "./cell.ts"; import { makeCertificates } from "./certs.ts"; import { traceChecks } from "./checks.ts"; import { dockerFreeGb, labelArgs, lowDisk, removeRun } from "./cleanup.ts"; @@ -53,7 +53,7 @@ import { import { readingChecks, refusedReadingChecks } from "./readings.ts"; import { cellId, renderReport } from "./report.ts"; import { runWithRetry } from "./retry.ts"; -import { inlineFiles, lintScenario } from "./scenario.ts"; +import { inlineFiles, lintScenario, missingFixture } from "./scenario.ts"; import { assertTrace } from "./traces.ts"; import type { Cell, Check, KnownFailure, Manifest, Registry, RegistryEntry, Role, Scenario } from "./types.ts"; @@ -172,6 +172,11 @@ export function loadRegistry(file = path.join(INTEROP, "registry.json")): Regist return registry; } +/** A live Company has no fixture hook: the lane's one greeting, under its registry entry's `fixture` (spec §10). */ +export function liveFixture(entry: RegistryEntry, domain: string): Record { + return withWebPage({ prompts: { hello: LIVE_GREETING }, ...(entry.fixture ?? {}) }, domain); +} + /** Live peers have no adapter manifest. Their declared version must be explicit. */ export function liveCompanyManifest(entry: RegistryEntry, scenarios: readonly Scenario[]): Manifest { const ids = scenarios.map((s) => s.id); @@ -353,7 +358,7 @@ export function runExitCode(cells: readonly Cell[], bad: readonly Cell[], stale: return bad.length > 0 || stale.length > 0 || (cells.length > 0 && !ran) || (requireSupported && cells.some((cell) => cell.outcome === "unsupported")) ? 1 : 0; } -async function runCells(options: { +export async function runCells(options: { scenarios: readonly Scenario[]; agents: readonly Implementation[]; companies: readonly Implementation[]; @@ -398,8 +403,14 @@ async function runCells(options: { cell.specificationRevision = protocolRevision(cell.protocolVersion!); const missing = [agent, company].filter((impl) => !impl.manifest!.scenarios.includes(scenario.id)).map((impl) => impl.entry.name); const hooks = (scenario.needs?.company ?? []).filter((hook) => !company.hooks.includes(hook)); - if (missing.length || hooks.length) { - cell.reason = missing.length ? `${missing.join(" and ")} don't list ${scenario.id}` : `${company.entry.name} has no hook ${hooks.join(", ")}`; + // A live Company's fixture is its registry entry's, known now: a scenario that needs a value it lacks can't run there (spec §10). + const lacking = company.entry.build ? null : missingFixture(scenario, withScopes(liveFixture(company.entry, company.domain))); + if (missing.length || hooks.length || lacking) { + cell.reason = missing.length + ? `${missing.join(" and ")} don't list ${scenario.id}` + : hooks.length + ? `${company.entry.name} has no hook ${hooks.join(", ")}` + : `${company.entry.name}'s registry entry has no ${lacking}`; options.log(`${scenario.id.padEnd(4)} ${cell.agent} × ${cell.company}: unsupported (${cell.reason})`); continue; } @@ -436,8 +447,7 @@ export async function runOne( const answer = await control(company.target, "POST", "/hooks/fixture", {}); if (answer.status === 200) fixture = answer.body; } - // A live Company has no fixture hook; the live scenarios need only a greeting, which says it is a test (spec §10). - if (!company.entry.build) fixture = { prompts: { hello: LIVE_GREETING }, ...(company.entry.fixture ?? {}), ...(fixture as Record) }; + if (!company.entry.build) fixture = { ...liveFixture(company.entry, company.domain), ...(fixture as Record) }; fixture = withWebPage(fixture, company.domain); const callbacks: string[] = []; @@ -484,7 +494,7 @@ export async function runOne( } }; - const run = await runCell(scenario, fixture, driver); + const run = await runCell(scenario, fixture, driver, { live: !company.entry.build }); cell.steps = run.steps; cell.outcome = run.outcome; if (run.reason) cell.reason = run.reason; diff --git a/interop/runner/scenario.ts b/interop/runner/scenario.ts index 992914f..d8524af 100644 --- a/interop/runner/scenario.ts +++ b/interop/runner/scenario.ts @@ -25,33 +25,44 @@ export function lookup(path: string, saved: Saved): unknown { return value; } -/** - * Replace references in a value. A string that is exactly one reference - * becomes the referenced value, whatever its type; references inside a - * longer string are spliced in as text. With `strict`, a reference to - * nothing is an error (the step can't run without it). - */ -export function interpolate(value: unknown, saved: Saved, strict = true): unknown { +type ResolveReference = (path: string) => unknown; + +/** Interpolation visits values, including nested arrays, but never object keys. */ +function replaceReferences(value: unknown, resolve: ResolveReference): unknown { if (typeof value === "string") { const whole = WHOLE.exec(value); - if (whole) { - const found = lookup(whole[1]!, saved); - if (found === undefined && strict) throw new ScenarioError(`\${${whole[1]}} refers to nothing`); - return found; - } + if (whole) return resolve(whole[1]!); return value.replace(PART, (_match, path: string) => { - const found = lookup(path, saved); - if (found === undefined && strict) throw new ScenarioError(`\${${path}} refers to nothing`); + const found = resolve(path); return typeof found === "string" ? found : JSON.stringify(found); }); } - if (Array.isArray(value)) return value.map((item) => interpolate(item, saved, strict)); + if (Array.isArray(value)) return value.map((item) => replaceReferences(item, resolve)); if (value !== null && typeof value === "object") { - return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, interpolate(item, saved, strict)])); + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, replaceReferences(item, resolve)])); } return value; } +function referenceLookup(saved: Saved, strict: boolean): ResolveReference { + return path => { + const found = lookup(path, saved); + if (found === undefined && strict) throw new ScenarioError(`\${${path}} refers to nothing`); + return found; + }; +} + +/** Whole references keep their type; embedded references become text. Strict missing values are errors. */ +export function interpolate(value: unknown, saved: Saved, strict = true): unknown { + return replaceReferences(value, referenceLookup(saved, strict)); +} + +/** Both execution and fixture preflight follow at most two interpolation passes. */ +function resolveReferences(value: unknown, resolve: ResolveReference): unknown { + const once = replaceReferences(value, resolve); + return JSON.stringify(once ?? null).includes("${") ? replaceReferences(once, resolve) : once; +} + /** * A step's params with every reference resolved, including references a * value brought in carries: a Company's fixture may say what an `api` step @@ -60,8 +71,7 @@ export function interpolate(value: unknown, saved: Saved, strict = true): unknow * keep pulling in references. */ export function resolveParams(value: unknown, saved: Saved): unknown { - const once = interpolate(value, saved); - return JSON.stringify(once ?? null).includes("${") ? interpolate(once, saved) : once; + return resolveReferences(value, referenceLookup(saved, true)); } /** @@ -241,6 +251,39 @@ export function actorOf(step: Step): { actor: "agent" | "company" | "runner" | " throw new ScenarioError("A step names no actor"); } +/** + * The first statically required fixture value that is absent, or null. + * Fixture-known conditions follow execution; runtime conditions are deferred. + * Optional steps still require their params, since only a 501 is optional. + * The runner's `first` resolves leniently and is checked during execution. + */ +export function missingFixture(scenario: Scenario, fixture: unknown): string | null { + const saved = { fixture }; + const fixturePath = (path: string) => path === "fixture" || path.startsWith("fixture."); + let missing: string | null = null; + const resolve: ResolveReference = reference => { + const path = reference.trim(); + // Earlier step results are unknown during planning; preserve their placeholders. + // A pass-two expression containing a preserved placeholder also depends on runtime data. + if (!fixturePath(path) || path.includes("${")) return `\${${reference}}`; + const found = lookup(path, saved); + if (found === undefined) missing ??= path; + return found; + }; + for (const step of scenario.steps) { + if (step.runner === "first" || step.expect) continue; + if (step.when !== undefined) { + const path = WHOLE.exec(step.when)?.[1]?.trim(); + if (!path || !fixturePath(path)) continue; + const condition = interpolate(step.when, saved, false); + if (condition === undefined || condition === null || condition === false) continue; + } + resolveReferences(step.params ?? {}, resolve); + if (missing) return missing; + } + return null; +} + /** Problems with a scenario file, empty when it is well formed. */ export function lintScenario(value: unknown, file: string): string[] { const problems: string[] = []; @@ -269,6 +312,7 @@ export function lintScenario(value: unknown, file: string): string[] { scenario.steps.forEach((step, index) => { const actors = ["agent", "company", "runner", "expect"].filter((key) => key in (step as object)); if (actors.length !== 1) problems.push(`${file}: step ${index} must name exactly one of agent, company, runner or expect`); + if (step.offers !== undefined && (step.offers !== true || !step.expect)) problems.push(`${file}: step ${index} offers must be true, on an expect step`); if (step.when !== undefined && (typeof step.when !== "string" || !/^\$\{[^}]+\}$/.test(step.when))) { problems.push(`${file}: step ${index} when must be one \${...} reference`); } diff --git a/interop/runner/types.ts b/interop/runner/types.ts index 938ce54..399a41e 100644 --- a/interop/runner/types.ts +++ b/interop/runner/types.ts @@ -81,6 +81,13 @@ export type Step = { /** A `${...}` reference: the step runs only when it names something other than null or false. */ readonly when?: string; readonly expect?: Expectations; + /** + * On an `expect` step: it states what the Company must offer for the + * scenario to apply (conversations, an MCP API). A live Company claims + * nothing (spec §10), so a miss makes its cell unsupported; a Company with + * an adapter claims the scenario in its manifest, so a miss fails. + */ + readonly offers?: true; readonly note?: string; }; diff --git a/interop/scenarios/C.json b/interop/scenarios/C.json index 9bf9434..3329579 100644 --- a/interop/scenarios/C.json +++ b/interop/scenarios/C.json @@ -5,7 +5,7 @@ "needs": { "agent": ["discover", "send", "read"] }, "steps": [ { "agent": "discover", "save": "d" }, - { "expect": { "${d.conversations}": true } }, + { "expect": { "${d.conversations}": true }, "offers": true }, { "agent": "send", "params": { "text": "${fixture.prompts.hello}" }, "save": "m" }, { "expect": { "${m.conversation}": { "$exists": true }, "${m.message_id}": { "$exists": true } } }, { "agent": "read", "params": { "conversation": "${m.conversation}", "wait": 2 }, "save": "r" }, diff --git a/interop/scenarios/CL.json b/interop/scenarios/CL.json index 275e049..e226435 100644 --- a/interop/scenarios/CL.json +++ b/interop/scenarios/CL.json @@ -6,7 +6,7 @@ "needs": { "agent": ["discover", "send", "read", "close"] }, "steps": [ { "agent": "discover", "save": "d" }, - { "expect": { "${d.conversations}": true } }, + { "expect": { "${d.conversations}": true }, "offers": true }, { "agent": "send", "params": { "text": "${fixture.prompts.hello}" }, "save": "m", "note": "In the live lane the greeting says it is a test and needs no action (cli.ts LIVE_GREETING)." }, { "expect": { "${m.conversation}": { "$exists": true }, "${m.message_id}": { "$exists": true } } }, { "agent": "read", "params": { "conversation": "${m.conversation}", "wait": 2 }, "save": "r" }, diff --git a/interop/scenarios/ML.json b/interop/scenarios/ML.json index 3217a5c..a2fadb8 100644 --- a/interop/scenarios/ML.json +++ b/interop/scenarios/ML.json @@ -19,7 +19,8 @@ "${d.apis}": { "$contains": "mcp" } - } + }, + "offers": true }, { "agent": "mcp_list", diff --git a/interop/test/runner.test.ts b/interop/test/runner.test.ts index 8ac03bb..6237060 100644 --- a/interop/test/runner.test.ts +++ b/interop/test/runner.test.ts @@ -8,12 +8,13 @@ import { upstreamFor } from "../proxy/routing.mjs"; import { VIOLATION_IDS } from "../../test/mock-company/violations.ts"; import { certificateNames, makeCertificates } from "../runner/certs.ts"; import { TRACE_CHECKS } from "../runner/checks.ts"; -import { gridCoverage, liveCompanyManifest, imageOf, loadKnownFailures, loadRegistry, loadScenarios, negotiateVersion, parseOptions, runExitCode, runOne, selected } from "../runner/cli.ts"; +import { gridCoverage, liveCompanyManifest, liveFixture, LIVE_GREETING, imageOf, loadKnownFailures, loadRegistry, loadScenarios, negotiateVersion, parseOptions, runCells, runExitCode, runOne, selected } from "../runner/cli.ts"; import { PROTOCOL_VERSIONS } from "../../src/poppy/protocol-version.ts"; import { dockerAsync, mapConcurrent, proxyAliases } from "../runner/docker.ts"; import { documentChecks, recordedFetcher } from "../runner/documents.ts"; import { cellId, renderReport } from "../runner/report.ts"; -import type { Cell } from "../runner/types.ts"; +import { lintScenario, missingFixture } from "../runner/scenario.ts"; +import type { Cell, Scenario } from "../runner/types.ts"; import { recording } from "./helpers.ts"; describe("options", () => { @@ -348,3 +349,74 @@ it("stops a live version mismatch before a Session request can send credentials" expect(calls).toEqual(["discover"]); } finally { rmSync(dir, { recursive: true, force: true }); } }); + +it("builds live fixtures consistently without depending on retired registry peers", () => { + const entry = liveEntry("retail", { fixture: { openapi_public: { api: "openapi", operation: "get", args: {} } } }); + const fixture = liveFixture(entry, "retail.example"); + const publicRead: Scenario = { id: "PUBLIC", title: "public read", steps: [ + { agent: "api", params: { api: "${fixture.openapi_public.api}", operation: "${fixture.openapi_public.operation}", args: "${fixture.openapi_public.args}" } } + ] }; + expect(missingFixture(publicRead, fixture)).toBeNull(); + expect(missingFixture(publicRead, liveFixture(liveEntry("no-api"), "no-api.example"))).toBe("fixture.openapi_public.api"); + expect(fixture).toMatchObject({ prompts: { hello: LIVE_GREETING }, web_page: "https://retail.example/" }); + const C = loadScenarios().find((scenario) => scenario.id === "C")!; + expect(missingFixture(C, liveFixture(liveEntry("conversation"), "conversation.example"))).toBeNull(); +}); + +it.each(["nested reference", "embedded reference", "optional step", "active condition"])("rejects a live %s before any adapter call", async mode => { + const original = loadScenarios().find(scenario => scenario.id === "S0")!; + const fixture = { + enabled: true, + openapi_public: { api: "openapi", operation: "GET /catalog", args: { account: mode === "embedded reference" ? "account=${fixture.missing}" : "${fixture.missing}" } } + }; + const scenario: Scenario = mode === "optional step" || mode === "active condition" + ? { ...original, steps: original.steps.map((step, index) => index === 3 + ? { ...step, ...(mode === "optional step" ? { optional: true } : { when: "${fixture.enabled}" }), params: { api: "openapi", operation: "GET /catalog", args: { account: "${fixture.missing}" } } } + : step) } + : original; + if (mode === "optional step" || mode === "active condition") fixture.openapi_public.args = { account: "available" }; + expect(lintScenario(scenario, "S0.json")).toEqual([]); + const agentEntry = loadRegistry().implementations.find(entry => entry.name === "pda")!; + const implementation = (entry: RegistryEntry) => ({ + entry, target: entry.name, domain: "synthetic.example", hooks: [], broken: null, + manifest: { interop: 1, name: entry.name, role: entry.role, commit: "test", poppy: ["0.4"], scenarios: [scenario.id] } + }); + const calls: string[] = []; + const cells = await runCells({ + scenarios: [scenario], agents: [implementation(agentEntry)], companies: [implementation(liveEntry("synthetic", { protocolVersion: "0.4", fixture }))], + lane: "live", proxy: null, resultsDir: tmpdir(), retry: false, log: () => {}, + control: async (_target, _method, pathname) => { + calls.push(pathname); + return { status: 200, body: { ok: true, result: { poppy_version: "0.4", signed_in: false, session_id: "synthetic" } } }; + } + }); + expect(cells).toHaveLength(1); + expect(cells[0]).toMatchObject({ outcome: "unsupported", steps: [] }); + expect(cells[0]!.reason).toContain("fixture.missing"); + expect(calls).toEqual([]); +}); + +it("executes a complete live fixture while leaving earlier-result references for runtime", async () => { + const scenario = loadScenarios().find(scenario => scenario.id === "S0")!; + const agent = loadRegistry().implementations.find(entry => entry.name === "pda")!; + const company = liveEntry("synthetic", { protocolVersion: "0.4", fixture: { + openapi_public: { api: "openapi", operation: "GET /catalog", args: { account: "${s.session_id}" } } + } }); + const implementation = (entry: RegistryEntry) => ({ + entry, target: entry.name, domain: "synthetic.example", hooks: [], broken: null, + manifest: { interop: 1, name: entry.name, role: entry.role, commit: "test", poppy: ["0.4"], scenarios: [scenario.id] } + }); + const apiArgs: unknown[] = []; + const cells = await runCells({ + scenarios: [scenario], agents: [implementation(agent)], companies: [implementation(company)], + lane: "live", proxy: null, resultsDir: tmpdir(), retry: false, log: () => {}, + control: async (_target, _method, pathname, body) => { + if (pathname !== "/step") return { status: 200, body: {} }; + const { op, params } = body as { op: string; params: { args?: unknown } }; + if (op === "api") apiArgs.push(params.args); + return { status: 200, body: { ok: true, result: { poppy_version: "0.4", signed_in: false, session_id: "synthetic", status: 200 } } }; + } + }); + expect(cells[0]!.outcome).toBe("pass"); + expect(apiArgs).toEqual([{ account: "synthetic" }, { account: "synthetic" }]); +}); diff --git a/interop/test/scenario.test.ts b/interop/test/scenario.test.ts index eeb25ba..7b92471 100644 --- a/interop/test/scenario.test.ts +++ b/interop/test/scenario.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; import { runCell, toReply, withScopes, type Driver, type Reply } from "../runner/cell.ts"; import { loadScenarios } from "../runner/cli.ts"; -import { compare, evaluate, expectedRefusals, firstOf, inlineFiles, interpolate, lintScenario, lookup, resolveParams } from "../runner/scenario.ts"; +import { compare, evaluate, expectedRefusals, firstOf, inlineFiles, interpolate, lintScenario, lookup, missingFixture, resolveParams } from "../runner/scenario.ts"; import type { Scenario } from "../runner/types.ts"; describe("interpolation", () => { @@ -154,6 +154,86 @@ describe("a cell", () => { expect(inapplicable.outcome).toBe("unsupported"); }); + it.each(["C", "CL"])("is unsupported when a live Company lacks conversations for %s, and fails when an adapter claimed it", async id => { + const C = loadScenarios().find(scenario => scenario.id === id)!; + const noConversations = () => driver({ discover: { kind: "ok", result: { conversations: false } } }); + const live = await runCell(C, {}, noConversations(), { live: true }); + expect(live.outcome).toBe("unsupported"); + expect(live.reason).toContain(`doesn't offer what ${id} tests`); + expect((await runCell(C, {}, noConversations())).outcome).toBe("fail"); + // An expectation not marked `offers` still fails against a live Company. + const S: Scenario = { id: "S0", title: "s", steps: [{ agent: "session", save: "s" }, { expect: { "${s.session_id}": { $exists: true } } }] }; + expect((await runCell(S, {}, driver({ session: { kind: "ok", result: {} } }), { live: true })).outcome).toBe("fail"); + }); + + it("keeps real MCP failures distinct from a live Company without MCP", async () => { + const ML = loadScenarios().find(scenario => scenario.id === "ML")!; + const absent = () => driver({ discover: { kind: "ok", result: { apis: [] } } }); + expect((await runCell(ML, {}, absent(), { live: true })).outcome).toBe("unsupported"); + expect((await runCell(ML, {}, absent())).outcome).toBe("fail"); + const brokenListing = driver({ discover: { kind: "ok", result: { apis: ["mcp"] } }, mcp_list: { kind: "ok", result: { tools: [] } } }); + expect((await runCell(ML, {}, brokenListing, { live: true })).outcome).toBe("fail"); + }); + + it("checks interpolated fixture values while leaving literal parameter keys alone", () => { + const literalKey: Scenario = { id: "KEY", title: "literal key", steps: [{ agent: "api", params: { "${fixture.not_a_reference}": "literal" } }] }; + expect(resolveParams(literalKey.steps[0]!.params, { fixture: {} })).toEqual({ "${fixture.not_a_reference}": "literal" }); + expect(missingFixture(literalKey, {})).toBeNull(); + const nestedValue: Scenario = { id: "VALUE", title: "nested value", steps: [{ agent: "api", params: { nested: [{ value: "${fixture.required}" }] } }] }; + expect(missingFixture(nestedValue, {})).toBe("fixture.required"); + }); + + it("finds the fixture value a scenario's steps can't run without, skipping steps that may go without", () => { + const publicRead: Scenario = { id: "PUBLIC", title: "public read", steps: [ + { agent: "api", params: { api: "${fixture.openapi_public.api}", operation: "${fixture.openapi_public.operation}", args: "${fixture.openapi_public.args}" } } + ] }; + expect(missingFixture(publicRead, { prompts: { hello: "Hello" } })).toBe("fixture.openapi_public.api"); + expect(missingFixture(publicRead, { openapi_public: { api: "openapi", operation: "get", args: {} } })).toBeNull(); + const lenient: Scenario = { + id: "W", + title: "w", + steps: [ + { agent: "api", params: { x: "${fixture.optional}" }, when: "${fixture.optional}" }, + { company: "require_nonce", optional: true }, + { runner: "first", params: { of: ["${fixture.a}", "${fixture.b}"] } }, + { agent: "send", params: { text: "say ${fixture.prompts.hello}" } } + ] + }; + expect(missingFixture(lenient, { prompts: { hello: "Hello" } })).toBeNull(); + expect(missingFixture(lenient, {})).toBe("fixture.prompts.hello"); + }); + + it.each([true, 0, "", false, null, undefined])("checks parameters only when a fixture condition is active (%s)", enabled => { + const scenario: Scenario = { id: "WHEN", title: "fixture condition", steps: [ + { agent: "api", when: "${fixture.enabled}", params: { account: "${fixture.account}" } } + ] }; + const active = enabled !== undefined && enabled !== null && enabled !== false; + expect(missingFixture(scenario, { enabled })).toBe(active ? "fixture.account" : null); + }); + + it("defers runtime conditions and fixture references to earlier results without following a third interpolation pass", () => { + const scenario: Scenario = { id: "LATER", title: "runtime result", steps: [ + { agent: "api", save: "upload_id" }, + { agent: "api", params: { args: "${fixture.args}" } }, + { agent: "send", when: "${upload_id.held}", params: { account: "${fixture.runtime_only}" } } + ] }; + expect(lintScenario(scenario, "LATER.json")).toEqual([]); + const fixture = { args: { photo: "${upload_id}", chain: "${fixture.second}" }, second: "${fixture.third}" }; + expect(missingFixture(scenario, fixture)).toBeNull(); + expect(resolveParams(scenario.steps[1]!.params, { fixture, upload_id: "photo-1" })).toEqual({ args: { photo: "photo-1", chain: "${fixture.third}" } }); + }); + + it("defers a second-pass path assembled from an unknown runtime result", () => { + const scenario: Scenario = { id: "DYNAMIC", title: "runtime-built path", steps: [ + { agent: "api", save: "s" }, + { agent: "api", params: { value: "${fixture.prefix}${s.suffix}" } } + ] }; + const fixture = { prefix: "${fixture.", present: "available" }; + expect(lintScenario(scenario, "DYNAMIC.json")).toEqual([]); + expect(missingFixture(scenario, fixture)).toBeNull(); + expect(resolveParams(scenario.steps[1]!.params, { fixture, s: { suffix: "present}" } })).toEqual({ value: "available" }); + }); + it("signs in with the Company's own scopes from its fixture, and none when it names none", async () => { expect(withScopes({ accounts: ["a"] }).scopes).toEqual({ read: [], write: [] }); expect(withScopes({ scopes: { read: ["orders:read"], write: ["orders:read", "orders:write"] } }).scopes).toEqual({ read: ["orders:read"], write: ["orders:read", "orders:write"] }); @@ -275,3 +355,10 @@ describe("a cell", () => { expect(toReply(200, { result: 1 }, true).kind).toBe("broken"); }); }); + +it("allows offers only as true, on an expect step", () => { + const base = { id: "ZZ", title: "z" }; + expect(lintScenario({ ...base, steps: [{ expect: { "${fixture.x}": true }, offers: true }] }, "ZZ.json")).toEqual([]); + expect(lintScenario({ ...base, steps: [{ agent: "discover", offers: true }] }, "ZZ.json").some((p) => /offers must be true, on an expect step/.test(p))).toBe(true); + expect(lintScenario({ ...base, steps: [{ expect: { "${fixture.x}": true }, offers: "yes" }] }, "ZZ.json").some((p) => /offers must be true/.test(p))).toBe(true); +});