From 98387c14fdbfa906c09fa17d7fec415e4758202d Mon Sep 17 00:00:00 2001 From: Amin Chirazi <32016576+AminChirazi@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:26:21 +0400 Subject: [PATCH] fix(trace): the published schema rejected traces flowproof itself writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The action `type` enum in trace-v1.schema.json listed thirteen of the fifteen actions the engine emits. `capture` and `set_checked` were missing, so any trace containing a `Remember …` or a `Check …` step failed validation against the schema this repository publishes as the contract. Nothing caught it, and the reason is the interesting part: the conformance test validates one fixture, and that fixture contained neither action. The schema was wrong for exactly as long as nothing exercised it — a gap that fails green, on the artifact external consumers validate against rather than on anything our own code reads. Both are in the enum now with their param shapes, and the fixture carries one of each — including the counted `capture` reading — so the enum cannot drift from the emitted actions again without a red test. Verified the other way too: with the old enum restored, the extended fixture fails. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 17 ++++++ .../schema/trace-v1.schema.json | 54 +++++++++++++++++++ .../tests/fixtures/sample.trace.jsonl | 3 ++ .../tests/schema_conformance.rs | 6 ++- docs/trace-format.md | 5 +- 5 files changed, 82 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e89aa41..0e8db61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,23 @@ together). The test now asserts the substance, that the reason names where containment IS enforced, rather than one wording of it. +- **The published schema rejected traces flowproof itself writes.** The + action `type` enum in `trace-v1.schema.json` listed thirteen of the fifteen + actions the engine emits. `capture` and `set_checked` were missing, so any + trace containing a `Remember …` or a `Check …` step failed validation + against the schema this repository publishes as the contract. + + Nothing caught it, and the reason is the interesting part: the conformance + test validates one fixture, and that fixture contained neither action. The + schema was wrong for exactly as long as nothing exercised it — a gap that + fails *green*, and on the artifact external consumers validate against + rather than on anything our own code reads. + + Both are in the enum now, with their param shapes, and the fixture carries + one of each — including the counted `capture` reading — so the enum cannot + drift from the emitted actions again without a red test. Verified the + other way too: with the old enum restored, the extended fixture fails. + - **Our own suite had a flaky test, and it was red-lighting `main`.** `seeded_fixture_mutation_survives_navigation` wrote two pages to disk and navigated between `file://` urls, asserting that a cart mutation survived diff --git a/crates/flowproof-trace/schema/trace-v1.schema.json b/crates/flowproof-trace/schema/trace-v1.schema.json index c1ebd40..0240268 100644 --- a/crates/flowproof-trace/schema/trace-v1.schema.json +++ b/crates/flowproof-trace/schema/trace-v1.schema.json @@ -416,6 +416,8 @@ "type_text", "press_key", "upload", + "capture", + "set_checked", "wait", "assert" ] @@ -425,6 +427,58 @@ } }, "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "capture" + } + } + }, + "then": { + "properties": { + "params": { + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Flow-scoped capture name. Only the NAME is stored: the value is read at execution time on record and on every replay." + }, + "count": { + "type": "boolean", + "description": "The counted reading - how many elements match, rather than one element's text. Absent means the text reading." + } + } + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "set_checked" + } + } + }, + "then": { + "properties": { + "params": { + "required": [ + "checked" + ], + "properties": { + "checked": { + "type": "boolean", + "description": "The STATE the control is driven to, not a toggle." + } + } + } + } + } + }, { "if": { "properties": { diff --git a/crates/flowproof-trace/tests/fixtures/sample.trace.jsonl b/crates/flowproof-trace/tests/fixtures/sample.trace.jsonl index c91e1b9..a4a6b4e 100644 --- a/crates/flowproof-trace/tests/fixtures/sample.trace.jsonl +++ b/crates/flowproof-trace/tests/fixtures/sample.trace.jsonl @@ -2,3 +2,6 @@ {"id":"s0001","intent":"Enter order type ZOR in the Order Type field","action":{"type":"type_text","params":{"text":"ZOR","submit":false}},"selectors":[{"tier":"native_id","provenance":"sap-com","confidence":1.0,"payload":{"id":"wnd[0]/usr/ctxtVBAK-AUART"}},{"tier":"structural","provenance":"uia","payload":{"path":[{"control_type":"Window","index":0},{"control_type":"Edit","index":3}]}},{"tier":"text_anchor","provenance":"vision","payload":{"text":"Order Type","relation":"right_of","max_distance_px":220}},{"tier":"visual_template","provenance":"vision","payload":{"template":"sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","region":[412,318,180,24]}},{"tier":"ai_relocation","provenance":"vision","payload":{"context":"The Order Type input in the Create Sales Order header section"}}],"sync":{"pre":[{"kind":"element_exists","selector_ref":0,"timeout_ms":10000}],"post":[{"kind":"ocr_text_present","text":"ZOR","region":[412,318,180,24],"timeout_ms":5000}]},"artifacts":{"pre_screenshot":"sha256:efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","post_screenshot":"sha256:0101010101010101010101010101010101010101010101010101010101010101","recording":{"start_ms":120,"end_ms":540}}} {"id":"s0002","intent":"Verify the order exists via the order API","action":{"type":"assert","params":{"kind":"api","request":{"method":"GET","url":"https://erp-qa.example.com/api/orders/latest"},"status":200}},"selectors":[{"tier":"ai_relocation","provenance":"vision","payload":{"context":"out-of-band assertion, no on-screen target"}}],"sync":{"pre":[],"post":[]},"artifacts":{}} {"id":"s0003","intent":"Test the connection via the API","action":{"type":"assert","params":{"kind":"api","request":{"method":"POST","url":"${API_BASE}/connections/test","body":{"provider":"postgres","connectionString":"${TEST_CONN_STRING}"},"headers":{"Authorization":"Bearer ${SESSION_TOKEN}"}},"status":200,"expect":{"body_contains":"Database not yet supported!"}}},"selectors":[{"tier":"ai_relocation","provenance":"vision","payload":{"context":"out-of-band assertion, no on-screen target"}}],"sync":{"pre":[],"post":[]},"artifacts":{}} +{"id": "s0004", "intent": "Remember the order total", "action": {"type": "capture", "params": {"name": "total"}}, "selectors": [{"tier": "native_id", "provenance": "web", "confidence": 1.0, "payload": {"id": "orderTotal"}}], "sync": {"pre": [{"kind": "element_exists", "selector_ref": 0, "timeout_ms": 10000}], "post": []}, "artifacts": {}} +{"id": "s0005", "intent": "Remember how many order rows appear", "action": {"type": "capture", "params": {"name": "rows", "count": true}}, "selectors": [{"tier": "native_id", "provenance": "web", "confidence": 1.0, "payload": {"css": ".order-row"}}], "sync": {"pre": [{"kind": "element_exists", "selector_ref": 0, "timeout_ms": 10000}], "post": []}, "artifacts": {}} +{"id": "s0006", "intent": "Accept the terms", "action": {"type": "set_checked", "params": {"checked": true}}, "selectors": [{"tier": "native_id", "provenance": "web", "confidence": 1.0, "payload": {"id": "terms"}}], "sync": {"pre": [{"kind": "element_exists", "selector_ref": 0, "timeout_ms": 10000}], "post": []}, "artifacts": {}} diff --git a/crates/flowproof-trace/tests/schema_conformance.rs b/crates/flowproof-trace/tests/schema_conformance.rs index a9f7bb5..38dd359 100644 --- a/crates/flowproof-trace/tests/schema_conformance.rs +++ b/crates/flowproof-trace/tests/schema_conformance.rs @@ -49,7 +49,11 @@ fn fixture_lines_parse_and_validate() { } // s0003 carries an api request body + headers with raw ${VAR} refs; // s0002 (headerless GET) proves the backward-compat default path. - assert_eq!(steps, 3, "fixture should contain three steps"); + // s0004/s0005 are the text and COUNTED capture readings and s0006 is + // `set_checked` - the three action types the schema did not list, which + // is exactly why this fixture has to carry one of each: the enum was + // wrong for as long as nothing here exercised them. + assert_eq!(steps, 6, "fixture should contain six steps"); } #[test] diff --git a/docs/trace-format.md b/docs/trace-format.md index 442c204..0434ccd 100644 --- a/docs/trace-format.md +++ b/docs/trace-format.md @@ -116,8 +116,9 @@ these fields existed) is byte-identical: - `intent` — the natural-language step description. Never executed; used for review, reporting, and as the prompt seed for `ai_relocation`/healing. - `action.type` — one of `launch`, `focus_window`, `click`, `double_click`, - `right_click`, `hover`, `drag`, `scroll`, `type_text`, `press_key`, `wait`, - `assert`. `params` is action-specific (see schema `$defs`). + `right_click`, `hover`, `drag`, `scroll`, `type_text`, `press_key`, + `upload`, `capture`, `set_checked`, `wait`, `assert`. `params` is + action-specific (see schema `$defs`). Text params (`type_text` text, assert expectations) may contain `${VAR}` **secret references**: the engine resolves them from the environment at execution time — recording and every replay — and the trace only ever