From 87ab377b728bccaed071fea013eb3113e2fb92e1 Mon Sep 17 00:00:00 2001 From: Sava Znatnov Date: Thu, 19 Nov 2026 01:52:26 +0300 Subject: [PATCH 1/3] fix(e2e): count a survey's answers by the listing the runner unwraps The runner turns a {items,truncated,next,total} listing into a list, so length(items) raised JMESPathTypeError and the nightly run stayed red. Co-Authored-By: Claude Opus 5.5 --- e2e/scenarios/forms/survey_options.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/scenarios/forms/survey_options.yaml b/e2e/scenarios/forms/survey_options.yaml index 6cedbd76..3d3a1d3e 100644 --- a/e2e/scenarios/forms/survey_options.yaml +++ b/e2e/scenarios/forms/survey_options.yaml @@ -50,11 +50,11 @@ steps: - id: answers run: forms answers list ${id} --ordering asc --page-size 1 --use-slugs --date-from 2020-01-01T00:00:00Z --date-to 2099-01-01T00:00:00Z --all expect: - "length(items)": 0 + "length(@)": 0 - id: answers-raw run: forms answers list ${id} --answer-format raw --all expect: - "length(items)": 0 + "length(@)": 0 # A form nobody answered: the columns are read all the same. - id: answer-columns run: forms answers columns-list ${id} --use-slugs From 26d9410cee3b63281aba6aa5d928bc587dedf7a4 Mon Sep 17 00:00:00 2001 From: Sava Znatnov Date: Thu, 19 Nov 2026 01:52:30 +0300 Subject: [PATCH 2/3] fix(e2e): name a timed-out step's command without the values the run learned subprocess.TimeoutExpired escaped the step and its traceback printed the rendered argv, uid included. The step now fails with its own error and the argv goes through hidden(). Co-Authored-By: Claude Opus 5.5 --- e2e/runner.py | 16 ++++++++++++++-- tests/tooling/test_e2e_runner.py | 22 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/e2e/runner.py b/e2e/runner.py index 113d2874..4357647f 100644 --- a/e2e/runner.py +++ b/e2e/runner.py @@ -264,7 +264,7 @@ def _run_step( ) -> None: where = f"[{scenario.name}/{step.id}]" arguments = render_command(step.run, variables) - completed = driver.run(arguments) + completed = _drive(where, driver, arguments, variables) retries = 0 while ( completed.exit_code != 0 @@ -274,7 +274,7 @@ def _run_step( ): time.sleep(step.retry.pause_seconds) retries += 1 - completed = driver.run(arguments) + completed = _drive(where, driver, arguments, variables) if retries and retried is not None and step.retry is not None: # Said whether it passed or not: a refusal that needed a second try is still a refusal. outcome = "passed" if completed.exit_code == 0 else "failed" @@ -310,6 +310,18 @@ def _run_step( variables[name] = str(value) +def _drive( + where: str, driver: Driver, arguments: Sequence[str], variables: dict[str, str] +) -> CommandResult: + """Run the step's command; a timeout is a step failure that shows the argv by name.""" + try: + return driver.run(arguments) + except subprocess.TimeoutExpired as error: + # The exception's own text is the rendered argv, with the values the run learned. + shown = hidden(shlex.join(arguments), variables) + raise ScenarioError(f"{where} `ycli {shown}` timed out after {error.timeout:g}s") from None + + def _parse(where: str, step: Step, completed: CommandResult) -> Any: stdout = completed.stdout if step.output == "bytes": diff --git a/tests/tooling/test_e2e_runner.py b/tests/tooling/test_e2e_runner.py index 6a32cab4..5f072144 100644 --- a/tests/tooling/test_e2e_runner.py +++ b/tests/tooling/test_e2e_runner.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import subprocess from typing import TYPE_CHECKING import pytest @@ -375,3 +376,24 @@ def test_a_step_that_expects_a_refusal_and_reads_its_output_is_no_scenario(besid """`exits` reads nothing of the output: what would be passed over is refused at load.""" with pytest.raises(ValueError, match="`expect` and `save` beside it would be passed over"): _scenario({"id": "a", "run": "gone", "exits": 9, **beside}) + + +class _TimingOut(Driver): + """Answers ``me`` with a uid and times out on every other command, as ``subprocess`` does.""" + + def run(self, arguments: Sequence[str]) -> CommandResult: + if arguments[0] == "me": + return CommandResult(0, json.dumps({"uid": "1130000012345678"}), "") + raise subprocess.TimeoutExpired(["ycli", *arguments], 120) + + +def test_a_step_that_times_out_fails_without_the_value_the_run_learned(): + scenario = _scenario( + {"id": "me", "run": "me get", "save": {"uid": "uid"}}, + {"id": "slow", "run": "users get ${uid}"}, + ) + with pytest.raises(ScenarioError) as failed: + run_scenario(scenario, _TimingOut(), {}) + assert str(failed.value) == "[s/slow] `ycli users get ` timed out after 120s" + assert "1130000012345678" not in str(failed.value) + assert failed.value.__suppress_context__ From 2f19f4e26b6a08a39340edc7ed31c6fa4d4b2a42 Mon Sep 17 00:00:00 2001 From: Sava Znatnov Date: Thu, 19 Nov 2026 02:41:21 +0300 Subject: [PATCH 3/3] fix(e2e): leave an unstamped e2e- object to its owner at a zero cutoff --older-than 0s counted a name with no run stamp as stale: the janitor printed it as skipped and then deleted it. Only an object that carries a stamp can age now. Co-Authored-By: Claude Opus 5.5 --- e2e/janitor.py | 6 +++++- tests/tooling/test_e2e_janitor.py | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/e2e/janitor.py b/e2e/janitor.py index 8bfd3afc..5f0e8529 100644 --- a/e2e/janitor.py +++ b/e2e/janitor.py @@ -42,6 +42,10 @@ def started_at(self) -> int | None: match = _STAMP.match(self.name) return int(match.group(1)) if match else None + def stale_by(self, cutoff: float) -> bool: + """Only a stamped object can age: ``Leftover("wiki", "1", "e2e-x").stale_by(1)`` -> no.""" + return self.started_at is not None and self.started_at <= cutoff + def parse_duration(text: str) -> int: """``parse_duration("6h") -> 21600`` (seconds); units s, m, h, d.""" @@ -130,7 +134,7 @@ def sweep( if runs is not None: leftovers = [item for item in leftovers if item.name.startswith(tuple(runs))] cutoff = now - older_than_seconds - stale = [item for item in leftovers if (item.started_at or now) <= cutoff] + stale = [item for item in leftovers if item.stale_by(cutoff)] unnamed = [item for item in leftovers if item.started_at is None] for item in unnamed: print(f"skipped {item.service} {item.identifier} {item.name!r}: no run stamp in the name") diff --git a/tests/tooling/test_e2e_janitor.py b/tests/tooling/test_e2e_janitor.py index 8dad963e..18c52822 100644 --- a/tests/tooling/test_e2e_janitor.py +++ b/tests/tooling/test_e2e_janitor.py @@ -73,6 +73,15 @@ def test_only_stale_run_named_objects_are_touched(capsys): assert "janitor: 5 e2e- objects, 3 stale, removed 3, 0 failed, 1 too recent" in output +def test_an_object_without_a_run_stamp_is_never_stale_even_at_a_zero_cutoff(capsys): + organization = FakeOrganization(CLOSE) + sweep(organization, "Q", 0, 200, dry_run=False, now=NOW) + assert ["wiki", "pages", "delete", "12"] not in organization.writes + output = capsys.readouterr().out + assert "skipped wiki 12 'e2e-handmade': no run stamp" in output + assert "deleted wiki 12" not in output + + def test_the_cap_holds_back_writes(capsys): organization = FakeOrganization(CLOSE) sweep(organization, "Q", 3600, 1, dry_run=False, now=NOW)