diff --git a/e2e/janitor.py b/e2e/janitor.py index 8bfd3afc..5f0e8529 100644 --- a/e2e/janitor.py +++ b/e2e/janitor.py @@ -42,6 +42,10 @@ def started_at(self) -> int | None: match = _STAMP.match(self.name) return int(match.group(1)) if match else None + def stale_by(self, cutoff: float) -> bool: + """Only a stamped object can age: ``Leftover("wiki", "1", "e2e-x").stale_by(1)`` -> no.""" + return self.started_at is not None and self.started_at <= cutoff + def parse_duration(text: str) -> int: """``parse_duration("6h") -> 21600`` (seconds); units s, m, h, d.""" @@ -130,7 +134,7 @@ def sweep( if runs is not None: leftovers = [item for item in leftovers if item.name.startswith(tuple(runs))] cutoff = now - older_than_seconds - stale = [item for item in leftovers if (item.started_at or now) <= cutoff] + stale = [item for item in leftovers if item.stale_by(cutoff)] unnamed = [item for item in leftovers if item.started_at is None] for item in unnamed: print(f"skipped {item.service} {item.identifier} {item.name!r}: no run stamp in the name") diff --git a/e2e/runner.py b/e2e/runner.py index 113d2874..4357647f 100644 --- a/e2e/runner.py +++ b/e2e/runner.py @@ -264,7 +264,7 @@ def _run_step( ) -> None: where = f"[{scenario.name}/{step.id}]" arguments = render_command(step.run, variables) - completed = driver.run(arguments) + completed = _drive(where, driver, arguments, variables) retries = 0 while ( completed.exit_code != 0 @@ -274,7 +274,7 @@ def _run_step( ): time.sleep(step.retry.pause_seconds) retries += 1 - completed = driver.run(arguments) + completed = _drive(where, driver, arguments, variables) if retries and retried is not None and step.retry is not None: # Said whether it passed or not: a refusal that needed a second try is still a refusal. outcome = "passed" if completed.exit_code == 0 else "failed" @@ -310,6 +310,18 @@ def _run_step( variables[name] = str(value) +def _drive( + where: str, driver: Driver, arguments: Sequence[str], variables: dict[str, str] +) -> CommandResult: + """Run the step's command; a timeout is a step failure that shows the argv by name.""" + try: + return driver.run(arguments) + except subprocess.TimeoutExpired as error: + # The exception's own text is the rendered argv, with the values the run learned. + shown = hidden(shlex.join(arguments), variables) + raise ScenarioError(f"{where} `ycli {shown}` timed out after {error.timeout:g}s") from None + + def _parse(where: str, step: Step, completed: CommandResult) -> Any: stdout = completed.stdout if step.output == "bytes": diff --git a/e2e/scenarios/forms/survey_options.yaml b/e2e/scenarios/forms/survey_options.yaml index 6cedbd76..3d3a1d3e 100644 --- a/e2e/scenarios/forms/survey_options.yaml +++ b/e2e/scenarios/forms/survey_options.yaml @@ -50,11 +50,11 @@ steps: - id: answers run: forms answers list ${id} --ordering asc --page-size 1 --use-slugs --date-from 2020-01-01T00:00:00Z --date-to 2099-01-01T00:00:00Z --all expect: - "length(items)": 0 + "length(@)": 0 - id: answers-raw run: forms answers list ${id} --answer-format raw --all expect: - "length(items)": 0 + "length(@)": 0 # A form nobody answered: the columns are read all the same. - id: answer-columns run: forms answers columns-list ${id} --use-slugs diff --git a/tests/tooling/test_e2e_janitor.py b/tests/tooling/test_e2e_janitor.py index 8dad963e..18c52822 100644 --- a/tests/tooling/test_e2e_janitor.py +++ b/tests/tooling/test_e2e_janitor.py @@ -73,6 +73,15 @@ def test_only_stale_run_named_objects_are_touched(capsys): assert "janitor: 5 e2e- objects, 3 stale, removed 3, 0 failed, 1 too recent" in output +def test_an_object_without_a_run_stamp_is_never_stale_even_at_a_zero_cutoff(capsys): + organization = FakeOrganization(CLOSE) + sweep(organization, "Q", 0, 200, dry_run=False, now=NOW) + assert ["wiki", "pages", "delete", "12"] not in organization.writes + output = capsys.readouterr().out + assert "skipped wiki 12 'e2e-handmade': no run stamp" in output + assert "deleted wiki 12" not in output + + def test_the_cap_holds_back_writes(capsys): organization = FakeOrganization(CLOSE) sweep(organization, "Q", 3600, 1, dry_run=False, now=NOW) diff --git a/tests/tooling/test_e2e_runner.py b/tests/tooling/test_e2e_runner.py index 6a32cab4..5f072144 100644 --- a/tests/tooling/test_e2e_runner.py +++ b/tests/tooling/test_e2e_runner.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import subprocess from typing import TYPE_CHECKING import pytest @@ -375,3 +376,24 @@ def test_a_step_that_expects_a_refusal_and_reads_its_output_is_no_scenario(besid """`exits` reads nothing of the output: what would be passed over is refused at load.""" with pytest.raises(ValueError, match="`expect` and `save` beside it would be passed over"): _scenario({"id": "a", "run": "gone", "exits": 9, **beside}) + + +class _TimingOut(Driver): + """Answers ``me`` with a uid and times out on every other command, as ``subprocess`` does.""" + + def run(self, arguments: Sequence[str]) -> CommandResult: + if arguments[0] == "me": + return CommandResult(0, json.dumps({"uid": "1130000012345678"}), "") + raise subprocess.TimeoutExpired(["ycli", *arguments], 120) + + +def test_a_step_that_times_out_fails_without_the_value_the_run_learned(): + scenario = _scenario( + {"id": "me", "run": "me get", "save": {"uid": "uid"}}, + {"id": "slow", "run": "users get ${uid}"}, + ) + with pytest.raises(ScenarioError) as failed: + run_scenario(scenario, _TimingOut(), {}) + assert str(failed.value) == "[s/slow] `ycli users get ` timed out after 120s" + assert "1130000012345678" not in str(failed.value) + assert failed.value.__suppress_context__