From 6b1ca154e656e97eb19275faa0d173904142a8a8 Mon Sep 17 00:00:00 2001 From: Amin Chirazi <32016576+AminChirazi@users.noreply.github.com> Date: Sat, 1 Aug 2026 12:44:39 +0400 Subject: [PATCH] feat(grammar): a count could be asserted but never used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `the "css:.row" appears 5 times` has always worked, and it is the right step when you know the answer. When the app decides the answer — a table built from whatever the backend returned — there was nothing to write. The number existed, the grammar could compare it to a literal, and no literal was available. `Remember how many "" appear as ` reads it into the same flow-scoped name a text capture uses, so everything captures already do keeps working, including `shows ${captured.rows} + 1`. The computed comparison's numeric normalisation is reused as it stands rather than re-parsed, which is what lets the two compose without either knowing about the other. Counting rides the ordinal every adapter already implements, so it means on each adapter what `the 2nd "Row"` means there; no adapter changed to gain it. The number is read at execution time on record and on every replay, so only the reference enters the trace. Zero fails. A selector typo matches nothing and so does an empty table, and `0` is a confident, plausible number to type into an application. The step refuses and names the one that means zero. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 35 ++++++++ crates/flowproof-agent/src/recorder.rs | 111 ++++++++++++++++++++++++- crates/flowproof-agent/src/rules.rs | 60 ++++++++++++- crates/flowproof-replay/src/lib.rs | 26 +++++- docs/authoring.md | 12 +++ docs/trace-format.md | 11 +++ 6 files changed, 248 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7edca40..cf21387 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,41 @@ together). ## Unreleased +### Added + +- **A count could be asserted but never used.** `the "css:.row" appears 5 + times` has always worked, and it is the right step when you know the + answer. When the app decides the answer — a table built from whatever the + backend returned — there was nothing to write. The number existed, the + grammar could compare it to a literal, and no literal was available. + + `Remember how many "" appear as ` reads it into the same + flow-scoped name a text capture uses, so everything captures already do + keeps working: + + ```yaml + - Remember how many "css:.order-row" appear as rows + - Type ${captured.rows} into the "Rowcount" field + - assert: the "Total" shows ${captured.rows} + 1 + ``` + + Counting rides the ordinal every adapter already implements, so it means + on each adapter exactly what `the 2nd "Row"` means there — no adapter + changed to gain it. The number is read at execution time on record and on + every replay, so only the reference enters the trace and a page that grew + a row does not need it rewritten. + + **Zero fails.** A selector typo matches nothing, and so does an empty + table — and `0` is a confident, plausible number to type into an + application. A capture that answered `0` to both would be the silent + wrong-value class again, so the step refuses and names the one that means + zero: `assert: the "" appears 0 times`. + + There is no second definition of what a number is: the computed + comparison's normalisation is reused as it stands, which is what lets + `shows ${captured.rows} + 1` compose without either side knowing about + the other. + ### Fixed - **The docs never said what a typed capture does with the text around diff --git a/crates/flowproof-agent/src/recorder.rs b/crates/flowproof-agent/src/recorder.rs index 6f1ddb2..e82ea1f 100644 --- a/crates/flowproof-agent/src/recorder.rs +++ b/crates/flowproof-agent/src/recorder.rs @@ -595,9 +595,20 @@ fn step_for(id: usize, intent: &str, app: &str, action: &ResolvedAction) -> Step selector_ref: Some(0), }), ), - ResolvedAction::Capture { target, name } => { + ResolvedAction::Capture { + target, + name, + count, + } => { let mut params = serde_json::Map::new(); params.insert("name".into(), name.clone().into()); + // A new PARAM KEY, not a new action: the payload is free-form, + // so a trace written before counting existed still loads, and + // an old reader meeting this simply does not understand the + // step rather than misreading it as a text capture. + if *count { + params.insert("count".into(), true.into()); + } (selectors_for(app, target, None), Action::Capture(params)) } ResolvedAction::AssertCaptured { @@ -1900,7 +1911,15 @@ pub fn record_with_reuse( | ResolvedAction::AssertChecked { .. } | ResolvedAction::AssertCaptured { .. } ); - if !is_assert { + // A COUNTING capture asks "how many", which is a question with + // a valid answer of none. The generic precheck would answer it + // with `ElementNotFound` - true, but it names a selector rather + // than the step that means zero, and this reading has its own + // account of why zero is refused. A TEXT capture keeps the + // precheck: it must read some element. + let is_counting_capture = + matches!(&action, ResolvedAction::Capture { count: true, .. }); + if !is_assert && !is_counting_capture { if let Some(selector) = &selector { if !driver.element_exists(selector)? { return Err(RecordError::ElementNotFound { @@ -1945,8 +1964,33 @@ pub fn record_with_reuse( ResolvedAction::SetChecked { checked, .. } => { driver.set_checked(targeted(), *checked)? } - ResolvedAction::Capture { name, .. } => { - let value = driver.read_text(targeted())?; + ResolvedAction::Capture { name, count, .. } => { + let value = if *count { + let found = flowproof_driver::count_matching( + driver, + targeted(), + flowproof_driver::COUNT_DIAGNOSTIC_CAP, + )?; + // Zero FAILS rather than remembering "0". A + // selector typo matches nothing, and so does an + // empty table - and a capture that answered `0` to + // both would type a confident wrong number into the + // app. The step that MEANS zero is an assertion, + // and the error says so. + if found == 0 { + return Err(RecordError::AssertMismatch { + intent: spec_step.intent().to_string(), + expected: "at least one matching element to count".to_string(), + actual: "nothing matched, so the count would be a guess - \ + to assert emptiness write 'assert: the \"\" \ + appears 0 times'" + .to_string(), + }); + } + found.to_string() + } else { + driver.read_text(targeted())? + }; captures.insert(name.clone(), value); } ResolvedAction::AssertCaptured { @@ -3322,6 +3366,65 @@ steps: /// step - `Click "Next" until the label changes` becomes ONE click - /// record green, and fail one replay in five. The decline has to /// outrank the fallback, and `calls == 0` is what proves it does. + /// A count is a READING, like any capture: taken at execution time on + /// record and on every replay, so a page that grew a row does not need + /// the trace rewritten. The number therefore never enters the trace - + /// only the reference does, exactly as a text capture works. + #[test] + fn a_counted_capture_records_the_reference_and_not_the_number() { + let spec = FlowSpec::parse( + "name: Rows\napp: web\nurl: https://e.test/x\nsteps:\n \ + - Remember how many \"css:.row\" appear as rows\n \ + - Type ${captured.rows} into the \"Count\" field\n", + ) + .expect("parses"); + let mut driver = MockAppDriver::new(&[".row", "Count"]).with_occurrences(".row", 7); + let out = std::env::temp_dir().join("flowproof-count-capture.trace.jsonl"); + record(&spec, &mut driver, &out).expect("records"); + + // The app was handed the count, not the reference. + assert!( + driver.typed.iter().any(|(_, text)| text == "7"), + "the resolved count must be typed: {:?}", + driver.typed + ); + let trace = std::fs::read_to_string(&out).expect("trace readable"); + assert!(trace.contains("\"count\":true"), "the reading is recorded"); + assert!( + trace.contains("${captured.rows}"), + "the typed step stores the REFERENCE" + ); + assert!( + !trace.contains("\"text\":\"7\""), + "the number itself must never enter the trace: {trace}" + ); + std::fs::remove_file(&out).ok(); + } + + /// The red path, and the reason this capture is not allowed to answer + /// `0`. A selector typo matches nothing and so does an empty table - + /// and `0` is a confident, plausible number to type into an app. The + /// step that MEANS zero is an assertion, and the error says so. + #[test] + fn a_count_of_zero_fails_instead_of_remembering_zero() { + let spec = FlowSpec::parse( + "name: Rows\napp: web\nurl: https://e.test/x\nsteps:\n \ + - Remember how many \"css:.typo\" appear as rows\n", + ) + .expect("parses"); + // `.typo` is not on the page - the shape of a selector mistake. + let mut driver = MockAppDriver::new(&[".row"]); + let out = std::env::temp_dir().join("flowproof-count-zero.trace.jsonl"); + let err = + record(&spec, &mut driver, &out).expect_err("a count of zero must fail the recording"); + let message = err.to_string(); + assert!( + message.contains("appears 0 times"), + "the failure must point at the step that MEANS zero: {message}" + ); + std::fs::remove_file(&out).ok(); + } + #[test] fn a_declined_shape_never_reaches_the_model_author() { for step in [ diff --git a/crates/flowproof-agent/src/rules.rs b/crates/flowproof-agent/src/rules.rs index befb3c2..ec437eb 100644 --- a/crates/flowproof-agent/src/rules.rs +++ b/crates/flowproof-agent/src/rules.rs @@ -201,7 +201,18 @@ pub enum ResolvedAction { /// Read a target's text into a flow-scoped name for later comparison /// (`Remember the "Balance" as balance`). The VALUE never enters the /// trace - only the name does, exactly like a `${VAR}` secret. - Capture { target: Target, name: String }, + /// Remember something about the target under `name`, for a later + /// assertion or a later `Type` to use. `count` picks WHICH reading: + /// false is the element's text, true is how many elements match. + /// + /// Either way the value is taken at execution time on record and on + /// every replay, so it never enters the trace - the same indirection + /// `${VAR}` secrets use. + Capture { + target: Target, + name: String, + count: bool, + }, /// Drive a checkbox-like control to a STATE (`Check`/`Uncheck`). /// Set-state rather than toggle, so the step means the same thing /// however the environment arrives: idempotent by design. @@ -2586,6 +2597,44 @@ mod web { // `Remember the [Nth ]"" as ` - read a value now so a // later assertion can compare against it. The value is read at // execution time on record AND replay, so it never enters the trace. + // `Remember how many "" appear as ` - the COUNT, not + // the text. Same family and the same indirection: the number is + // taken at execution time on record and on every replay, so a page + // that grew a row does not need the trace rewritten. + // + // Checked before `remember the` because "how many" is not a target. + if let Some(rest) = strip_prefix_ci(trimmed, "remember how many ") { + let tail = rest.trim(); + if let Some(quoted) = tail.strip_prefix('"') { + if let Some((label, after)) = quoted_label(quoted) { + let after = after.trim(); + // `appear` reads correctly for a plural count; `appears` + // is accepted because the singular slips out when the + // author is thinking of one row. + let after = strip_prefix_ci(after, "appear as ") + .or_else(|| strip_prefix_ci(after, "appears as ")); + if let Some(name) = after.map(str::trim) { + if !valid_capture_name(name) { + return Err(unresolvable( + trimmed, + "a capture name must start with a lowercase letter and \ + contain only lowercase letters, digits, and underscores", + )); + } + return Ok(vec![ResolvedAction::Capture { + target: target_from_label(label), + name: name.to_string(), + count: true, + }]); + } + } + } + return Err(unresolvable( + trimmed, + "expected 'Remember how many \"\" appear as '", + )); + } + if let Some(rest) = strip_prefix_ci(trimmed, "remember the ") { let (nth, tail) = split_ordinal(rest.trim()); if let Some(quoted) = tail.strip_prefix('"') { @@ -2602,6 +2651,7 @@ mod web { return Ok(vec![ResolvedAction::Capture { target: scoped_target(trimmed, nth, label, scope)?, name: name.to_string(), + count: false, }]); } } @@ -3868,6 +3918,10 @@ mod tests { // Interpolation: several references in one step, and literal // text around them. Documented under "A typed value is // interpolated, not evaluated". + ( + "web", + r#"Remember how many "css:.order-row" appear as rows"#, + ), ("web", r#"Type order-${captured.oid} into the "Ref" field"#), ( "web", @@ -3983,6 +4037,7 @@ mod tests { "web", r#"the "Amount" in the "css:[data-test=transaction]" containing "Invoice 4711" shows 50"#, ), + ("web", r#"the "Total" shows ${captured.rows} + 1"#), ("calc", "display shows 8"), ("notepad", "document contains hello"), ]; @@ -5376,7 +5431,8 @@ mod scoped_target_tests { .expect("parses"), vec![ResolvedAction::Capture { target: wanted.clone(), - name: "total".into() + name: "total".into(), + count: false }] ); assert_eq!( diff --git a/crates/flowproof-replay/src/lib.rs b/crates/flowproof-replay/src/lib.rs index 0187200..f11f4bd 100644 --- a/crates/flowproof-replay/src/lib.rs +++ b/crates/flowproof-replay/src/lib.rs @@ -1781,7 +1781,31 @@ fn execute_step( }; // Read at execution time, on replay exactly as on record - // which is why the value never needs to be in the trace. - let value = driver.read_text(&target)?; + // `count` picks the reading; the indirection is the same. + let value = if params.get("count").and_then(|v| v.as_bool()) == Some(true) { + let found = flowproof_driver::count_matching( + driver, + &target, + flowproof_driver::COUNT_DIAGNOSTIC_CAP, + )?; + // Unreachable while the target resolved, since a + // resolved element is a first match - but stated + // rather than assumed, because "0" is the one value + // this capture must never quietly hand on. + if found == 0 { + return Ok(( + Err( + "nothing matched, so the count would be a guess - to assert \ + emptiness use 'the \"\" appears 0 times'" + .to_string(), + ), + matched, + )); + } + found.to_string() + } else { + driver.read_text(&target)? + }; captures.insert(name.to_string(), value); (Ok(()), matched) } diff --git a/docs/authoring.md b/docs/authoring.md index 88bea32..1cfe3bc 100644 --- a/docs/authoring.md +++ b/docs/authoring.md @@ -48,6 +48,7 @@ before migrating a suite's setup helpers step by step. | `Replace the field with ` | | | `Clear the [2nd ]"