diff --git a/CHANGELOG.md b/CHANGELOG.md index e89aa41..0e8db61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,23 @@ together). The test now asserts the substance, that the reason names where containment IS enforced, rather than one wording of it. +- **The published schema rejected traces flowproof itself writes.** The + action `type` enum in `trace-v1.schema.json` listed thirteen of the fifteen + actions the engine emits. `capture` and `set_checked` were missing, so any + trace containing a `Remember …` or a `Check …` step failed validation + against the schema this repository publishes as the contract. + + Nothing caught it, and the reason is the interesting part: the conformance + test validates one fixture, and that fixture contained neither action. The + schema was wrong for exactly as long as nothing exercised it — a gap that + fails *green*, and on the artifact external consumers validate against + rather than on anything our own code reads. + + Both are in the enum now, with their param shapes, and the fixture carries + one of each — including the counted `capture` reading — so the enum cannot + drift from the emitted actions again without a red test. Verified the + other way too: with the old enum restored, the extended fixture fails. + - **Our own suite had a flaky test, and it was red-lighting `main`.** `seeded_fixture_mutation_survives_navigation` wrote two pages to disk and navigated between `file://` urls, asserting that a cart mutation survived diff --git a/crates/flowproof-trace/schema/trace-v1.schema.json b/crates/flowproof-trace/schema/trace-v1.schema.json index c1ebd40..0240268 100644 --- a/crates/flowproof-trace/schema/trace-v1.schema.json +++ b/crates/flowproof-trace/schema/trace-v1.schema.json @@ -416,6 +416,8 @@ "type_text", "press_key", "upload", + "capture", + "set_checked", "wait", "assert" ] @@ -425,6 +427,58 @@ } }, "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "capture" + } + } + }, + "then": { + "properties": { + "params": { + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Flow-scoped capture name. Only the NAME is stored: the value is read at execution time on record and on every replay." + }, + "count": { + "type": "boolean", + "description": "The counted reading - how many elements match, rather than one element's text. Absent means the text reading." + } + } + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "set_checked" + } + } + }, + "then": { + "properties": { + "params": { + "required": [ + "checked" + ], + "properties": { + "checked": { + "type": "boolean", + "description": "The STATE the control is driven to, not a toggle." + } + } + } + } + } + }, { "if": { "properties": { diff --git a/crates/flowproof-trace/tests/fixtures/sample.trace.jsonl b/crates/flowproof-trace/tests/fixtures/sample.trace.jsonl index c91e1b9..a4a6b4e 100644 --- a/crates/flowproof-trace/tests/fixtures/sample.trace.jsonl +++ b/crates/flowproof-trace/tests/fixtures/sample.trace.jsonl @@ -2,3 +2,6 @@ {"id":"s0001","intent":"Enter order type ZOR in the Order Type field","action":{"type":"type_text","params":{"text":"ZOR","submit":false}},"selectors":[{"tier":"native_id","provenance":"sap-com","confidence":1.0,"payload":{"id":"wnd[0]/usr/ctxtVBAK-AUART"}},{"tier":"structural","provenance":"uia","payload":{"path":[{"control_type":"Window","index":0},{"control_type":"Edit","index":3}]}},{"tier":"text_anchor","provenance":"vision","payload":{"text":"Order Type","relation":"right_of","max_distance_px":220}},{"tier":"visual_template","provenance":"vision","payload":{"template":"sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","region":[412,318,180,24]}},{"tier":"ai_relocation","provenance":"vision","payload":{"context":"The Order Type input in the Create Sales Order header section"}}],"sync":{"pre":[{"kind":"element_exists","selector_ref":0,"timeout_ms":10000}],"post":[{"kind":"ocr_text_present","text":"ZOR","region":[412,318,180,24],"timeout_ms":5000}]},"artifacts":{"pre_screenshot":"sha256:efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","post_screenshot":"sha256:0101010101010101010101010101010101010101010101010101010101010101","recording":{"start_ms":120,"end_ms":540}}} {"id":"s0002","intent":"Verify the order exists via the order API","action":{"type":"assert","params":{"kind":"api","request":{"method":"GET","url":"https://erp-qa.example.com/api/orders/latest"},"status":200}},"selectors":[{"tier":"ai_relocation","provenance":"vision","payload":{"context":"out-of-band assertion, no on-screen target"}}],"sync":{"pre":[],"post":[]},"artifacts":{}} {"id":"s0003","intent":"Test the connection via the API","action":{"type":"assert","params":{"kind":"api","request":{"method":"POST","url":"${API_BASE}/connections/test","body":{"provider":"postgres","connectionString":"${TEST_CONN_STRING}"},"headers":{"Authorization":"Bearer ${SESSION_TOKEN}"}},"status":200,"expect":{"body_contains":"Database not yet supported!"}}},"selectors":[{"tier":"ai_relocation","provenance":"vision","payload":{"context":"out-of-band assertion, no on-screen target"}}],"sync":{"pre":[],"post":[]},"artifacts":{}} +{"id": "s0004", "intent": "Remember the order total", "action": {"type": "capture", "params": {"name": "total"}}, "selectors": [{"tier": "native_id", "provenance": "web", "confidence": 1.0, "payload": {"id": "orderTotal"}}], "sync": {"pre": [{"kind": "element_exists", "selector_ref": 0, "timeout_ms": 10000}], "post": []}, "artifacts": {}} +{"id": "s0005", "intent": "Remember how many order rows appear", "action": {"type": "capture", "params": {"name": "rows", "count": true}}, "selectors": [{"tier": "native_id", "provenance": "web", "confidence": 1.0, "payload": {"css": ".order-row"}}], "sync": {"pre": [{"kind": "element_exists", "selector_ref": 0, "timeout_ms": 10000}], "post": []}, "artifacts": {}} +{"id": "s0006", "intent": "Accept the terms", "action": {"type": "set_checked", "params": {"checked": true}}, "selectors": [{"tier": "native_id", "provenance": "web", "confidence": 1.0, "payload": {"id": "terms"}}], "sync": {"pre": [{"kind": "element_exists", "selector_ref": 0, "timeout_ms": 10000}], "post": []}, "artifacts": {}} diff --git a/crates/flowproof-trace/tests/schema_conformance.rs b/crates/flowproof-trace/tests/schema_conformance.rs index a9f7bb5..38dd359 100644 --- a/crates/flowproof-trace/tests/schema_conformance.rs +++ b/crates/flowproof-trace/tests/schema_conformance.rs @@ -49,7 +49,11 @@ fn fixture_lines_parse_and_validate() { } // s0003 carries an api request body + headers with raw ${VAR} refs; // s0002 (headerless GET) proves the backward-compat default path. - assert_eq!(steps, 3, "fixture should contain three steps"); + // s0004/s0005 are the text and COUNTED capture readings and s0006 is + // `set_checked` - the three action types the schema did not list, which + // is exactly why this fixture has to carry one of each: the enum was + // wrong for as long as nothing here exercised them. + assert_eq!(steps, 6, "fixture should contain six steps"); } #[test] diff --git a/docs/trace-format.md b/docs/trace-format.md index 442c204..0434ccd 100644 --- a/docs/trace-format.md +++ b/docs/trace-format.md @@ -116,8 +116,9 @@ these fields existed) is byte-identical: - `intent` — the natural-language step description. Never executed; used for review, reporting, and as the prompt seed for `ai_relocation`/healing. - `action.type` — one of `launch`, `focus_window`, `click`, `double_click`, - `right_click`, `hover`, `drag`, `scroll`, `type_text`, `press_key`, `wait`, - `assert`. `params` is action-specific (see schema `$defs`). + `right_click`, `hover`, `drag`, `scroll`, `type_text`, `press_key`, + `upload`, `capture`, `set_checked`, `wait`, `assert`. `params` is + action-specific (see schema `$defs`). Text params (`type_text` text, assert expectations) may contain `${VAR}` **secret references**: the engine resolves them from the environment at execution time — recording and every replay — and the trace only ever