Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,23 @@ together).
The test now asserts the substance, that the reason names where
containment IS enforced, rather than one wording of it.

- **The published schema rejected traces flowproof itself writes.** The
action `type` enum in `trace-v1.schema.json` listed thirteen of the fifteen
actions the engine emits. `capture` and `set_checked` were missing, so any
trace containing a `Remember …` or a `Check …` step failed validation
against the schema this repository publishes as the contract.

Nothing caught it, and the reason is the interesting part: the conformance
test validates one fixture, and that fixture contained neither action. The
schema was wrong for exactly as long as nothing exercised it — a gap that
fails *green*, and on the artifact external consumers validate against
rather than on anything our own code reads.

Both are in the enum now, with their param shapes, and the fixture carries
one of each — including the counted `capture` reading — so the enum cannot
drift from the emitted actions again without a red test. Verified the
other way too: with the old enum restored, the extended fixture fails.

- **Our own suite had a flaky test, and it was red-lighting `main`.**
`seeded_fixture_mutation_survives_navigation` wrote two pages to disk and
navigated between `file://` urls, asserting that a cart mutation survived
Expand Down
54 changes: 54 additions & 0 deletions crates/flowproof-trace/schema/trace-v1.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,8 @@
"type_text",
"press_key",
"upload",
"capture",
"set_checked",
"wait",
"assert"
]
Expand All @@ -425,6 +427,58 @@
}
},
"allOf": [
{
"if": {
"properties": {
"type": {
"const": "capture"
}
}
},
"then": {
"properties": {
"params": {
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "Flow-scoped capture name. Only the NAME is stored: the value is read at execution time on record and on every replay."
},
"count": {
"type": "boolean",
"description": "The counted reading - how many elements match, rather than one element's text. Absent means the text reading."
}
}
}
}
}
},
{
"if": {
"properties": {
"type": {
"const": "set_checked"
}
}
},
"then": {
"properties": {
"params": {
"required": [
"checked"
],
"properties": {
"checked": {
"type": "boolean",
"description": "The STATE the control is driven to, not a toggle."
}
}
}
}
}
},
{
"if": {
"properties": {
Expand Down
3 changes: 3 additions & 0 deletions crates/flowproof-trace/tests/fixtures/sample.trace.jsonl
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,6 @@
{"id":"s0001","intent":"Enter order type ZOR in the Order Type field","action":{"type":"type_text","params":{"text":"ZOR","submit":false}},"selectors":[{"tier":"native_id","provenance":"sap-com","confidence":1.0,"payload":{"id":"wnd[0]/usr/ctxtVBAK-AUART"}},{"tier":"structural","provenance":"uia","payload":{"path":[{"control_type":"Window","index":0},{"control_type":"Edit","index":3}]}},{"tier":"text_anchor","provenance":"vision","payload":{"text":"Order Type","relation":"right_of","max_distance_px":220}},{"tier":"visual_template","provenance":"vision","payload":{"template":"sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","region":[412,318,180,24]}},{"tier":"ai_relocation","provenance":"vision","payload":{"context":"The Order Type input in the Create Sales Order header section"}}],"sync":{"pre":[{"kind":"element_exists","selector_ref":0,"timeout_ms":10000}],"post":[{"kind":"ocr_text_present","text":"ZOR","region":[412,318,180,24],"timeout_ms":5000}]},"artifacts":{"pre_screenshot":"sha256:efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","post_screenshot":"sha256:0101010101010101010101010101010101010101010101010101010101010101","recording":{"start_ms":120,"end_ms":540}}}
{"id":"s0002","intent":"Verify the order exists via the order API","action":{"type":"assert","params":{"kind":"api","request":{"method":"GET","url":"https://erp-qa.example.com/api/orders/latest"},"status":200}},"selectors":[{"tier":"ai_relocation","provenance":"vision","payload":{"context":"out-of-band assertion, no on-screen target"}}],"sync":{"pre":[],"post":[]},"artifacts":{}}
{"id":"s0003","intent":"Test the connection via the API","action":{"type":"assert","params":{"kind":"api","request":{"method":"POST","url":"${API_BASE}/connections/test","body":{"provider":"postgres","connectionString":"${TEST_CONN_STRING}"},"headers":{"Authorization":"Bearer ${SESSION_TOKEN}"}},"status":200,"expect":{"body_contains":"Database not yet supported!"}}},"selectors":[{"tier":"ai_relocation","provenance":"vision","payload":{"context":"out-of-band assertion, no on-screen target"}}],"sync":{"pre":[],"post":[]},"artifacts":{}}
{"id": "s0004", "intent": "Remember the order total", "action": {"type": "capture", "params": {"name": "total"}}, "selectors": [{"tier": "native_id", "provenance": "web", "confidence": 1.0, "payload": {"id": "orderTotal"}}], "sync": {"pre": [{"kind": "element_exists", "selector_ref": 0, "timeout_ms": 10000}], "post": []}, "artifacts": {}}
{"id": "s0005", "intent": "Remember how many order rows appear", "action": {"type": "capture", "params": {"name": "rows", "count": true}}, "selectors": [{"tier": "native_id", "provenance": "web", "confidence": 1.0, "payload": {"css": ".order-row"}}], "sync": {"pre": [{"kind": "element_exists", "selector_ref": 0, "timeout_ms": 10000}], "post": []}, "artifacts": {}}
{"id": "s0006", "intent": "Accept the terms", "action": {"type": "set_checked", "params": {"checked": true}}, "selectors": [{"tier": "native_id", "provenance": "web", "confidence": 1.0, "payload": {"id": "terms"}}], "sync": {"pre": [{"kind": "element_exists", "selector_ref": 0, "timeout_ms": 10000}], "post": []}, "artifacts": {}}
Comment on lines +5 to +7
6 changes: 5 additions & 1 deletion crates/flowproof-trace/tests/schema_conformance.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,11 @@ fn fixture_lines_parse_and_validate() {
}
// s0003 carries an api request body + headers with raw ${VAR} refs;
// s0002 (headerless GET) proves the backward-compat default path.
assert_eq!(steps, 3, "fixture should contain three steps");
// s0004/s0005 are the text and COUNTED capture readings and s0006 is
// `set_checked` - the three action types the schema did not list, which
// is exactly why this fixture has to carry one of each: the enum was
// wrong for as long as nothing here exercised them.
Comment on lines +52 to +55
assert_eq!(steps, 6, "fixture should contain six steps");
}

#[test]
Expand Down
5 changes: 3 additions & 2 deletions docs/trace-format.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,8 +116,9 @@ these fields existed) is byte-identical:
- `intent` — the natural-language step description. Never executed; used for
review, reporting, and as the prompt seed for `ai_relocation`/healing.
- `action.type` — one of `launch`, `focus_window`, `click`, `double_click`,
`right_click`, `hover`, `drag`, `scroll`, `type_text`, `press_key`, `wait`,
`assert`. `params` is action-specific (see schema `$defs`).
`right_click`, `hover`, `drag`, `scroll`, `type_text`, `press_key`,
`upload`, `capture`, `set_checked`, `wait`, `assert`. `params` is
action-specific (see schema `$defs`).
Text params (`type_text` text, assert expectations) may contain `${VAR}`
**secret references**: the engine resolves them from the environment at
execution time — recording and every replay — and the trace only ever
Expand Down
Loading