Budget e2e time per scenario, not as a fixed suite total

docs/operations.md

@@ -198,7 +198,8 @@ ran, and the memory and fact rows, read by SQL. Every `/name` the bot says is
 checked against the commands Discord offers where it said it.
 
 - `E2E_REQUIRE_DB=1` turns "no database" from a skip into a failure. CI sets it.
-- The suite fails the run if it takes more than 120 seconds.
+- The suite fails the run if any scenario takes more than 10 seconds, or the
+  whole suite more than 1.5 seconds per scenario on average.
 - `E2E_UPDATE_SNAPSHOTS=1` rewrites `tests/e2e/snapshots/commands.json`, the
   exact command payload the bot syncs; review the diff like code.
 - discord.py is pinned to `~=2.7.1` in the dev extras because the fake wire

openspec/changes/e2e-harness/tasks.md

@@ -37,7 +37,7 @@
 - [x] 4.4 Real Postgres: `TEST_DATABASE_URL` with a testcontainers pgvector fallback, `E2E_REQUIRE_DB=1` turning a skip into a failure, a TRUNCATE per test
 - [x] 4.5 Network canary: httpx's real transports raise inside the suite
 - [x] 4.6 discord.py pinned `~=2.7.1` in the dev extras; a canary test naming each internal the wire uses
-- [x] 4.7 A 60-second budget for the suite, enforced (since raised to 90 s, then 120 s)
+- [x] 4.7 A 60-second budget for the suite, enforced (since raised to 90 s, then 120 s; replaced by a 10 s per-scenario budget and a 1.5 s-per-scenario suite backstop, because a fixed total failed main as the suite grew)
 - [x] 4.8 CI step: `pytest tests/e2e` with `E2E_REQUIRE_DB=1` after the migration
 - [x] 4.9 Docs: end-to-end tests in `docs/operations.md`, including the rule that a production fix replays its transcript as a scenario
 

tests/e2e/conftest.py

@@ -34,10 +34,13 @@
 ROOT = Path(__file__).resolve().parents[2]
 DEFAULT_URL = "postgresql+asyncpg://chatmemory:chatmemory@localhost:5432/chatmemory"
 PGVECTOR_IMAGE = "pgvector/pgvector:pg17"
-# Raised from 60 s at 100 scenarios, when CI first ran 60.6 s with every test
-# green, and from 90 s at ~150, when CI ran 92.8 s with no scenario over 3 s.
-# The budget exists to catch one slow scenario, not a growing suite.
-BUDGET_SECONDS = 120.0
+# A scenario over this is the regression the budget exists to catch: one slow
+# test, named. The slowest today is under 3 s.
+TEST_BUDGET_SECONDS = 10.0
+# A backstop for the whole suite that grows with it. A fixed total (60 s, then
+# 90 s, then 120 s) failed main each time enough scenarios were added, with
+# every test passing and none of them slow.
+SUITE_SECONDS_PER_TEST = 1.5
 
 COLLEAGUE_CRYPTO = (
     "Our project treasury 0xD5C95aF87F6e1E83507AC96b2eE4484B9AFEbDd5 holds the "
@@ -192,22 +195,36 @@ async def bot(
 
 # --- the time budget ----------------------------------------------------------
 
-_spent = 0.0
+_durations: dict[str, float] = {}
+
+
+def over_budget(durations: dict[str, float]) -> list[str]:
+    """What broke the budget: each slow scenario, then the suite as a whole."""
+    problems = [
+        f"{node} took {spent:.1f}s, over the {TEST_BUDGET_SECONDS:.0f}s per-scenario budget"
+        for node, spent in sorted(durations.items())
+        if spent > TEST_BUDGET_SECONDS
+    ]
+    total, allowed = sum(durations.values()), SUITE_SECONDS_PER_TEST * len(durations)
+    if total > allowed:
+        problems.append(
+            f"end-to-end suite took {total:.1f}s, over its {allowed:.0f}s budget "
+            f"({SUITE_SECONDS_PER_TEST}s x {len(durations)} scenarios)"
+        )
+    return problems
 
 
 def pytest_runtest_logreport(report: pytest.TestReport) -> None:
-    global _spent
     if report.nodeid.startswith("tests/e2e/"):
-        _spent += report.duration
+        _durations[report.nodeid] = _durations.get(report.nodeid, 0.0) + report.duration
 
 
 def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
     """The budget is enforced, not hoped for: over it, the run fails."""
-    if _spent > BUDGET_SECONDS:
+    problems = over_budget(_durations)
+    if problems:
         reporter = session.config.pluginmanager.get_plugin("terminalreporter")
         if reporter is not None:
-            reporter.write_line(
-                f"end-to-end suite took {_spent:.1f}s, over its {BUDGET_SECONDS:.0f}s budget",
-                red=True,
-            )
+            for line in problems:
+                reporter.write_line(line, red=True)
         session.exitstatus = pytest.ExitCode.TESTS_FAILED

tests/integration/test_market_live.py

@@ -23,6 +23,7 @@
 import pytest
 import pytest_asyncio
 
+from chatmemory.adapters.market.coingecko import COINGECKO_ENDPOINT
 from chatmemory.adapters.market.frankfurter import FRANKFURTER_ENDPOINT
 from chatmemory.adapters.market.registration import MarketToolsConfig, build_market_tools
 from chatmemory.adapters.mcp_client.client import connect
@@ -57,6 +58,24 @@ async def online() -> AsyncIterator[None]:
     yield
 
 
+#: Statuses CoinGecko's keyless tier answers when it refuses a caller outright
+#: on the price endpoint, while /ping still answers 200 (a CI runner's shared
+#: IP, or a burst). Our code cannot be tested against a
+#: provider that will not answer, so the crypto test skips rather than fails.
+PROVIDER_REFUSED = frozenset({403, 429})
+
+
+@pytest_asyncio.fixture
+async def coingecko(online: None) -> AsyncIterator[None]:
+    async with httpx.AsyncClient(timeout=5.0) as client:
+        ping = await client.get(
+            COINGECKO_ENDPOINT, params={"ids": "bitcoin", "vs_currencies": "usd"}
+        )
+    if ping.status_code in PROVIDER_REFUSED:
+        pytest.skip(f"CoinGecko refuses this runner: HTTP {ping.status_code}")
+    yield
+
+
 class Governed:
     """One federation for a test, so a cached figure is reused as in the bot."""
 
@@ -106,7 +125,7 @@ async def ask(
         return await run.ask(question, tool, arguments)
 
 
-async def test_live_btc_and_eth_carry_their_quote_time(online: None) -> None:
+async def test_live_btc_and_eth_carry_their_quote_time(coingecko: None) -> None:
     """ETH asked about as "ether" is not refused, and each price is dated.
 
     One federation for both, as in the running bot: CoinGecko's keyless tier

tests/unit/test_e2e_budget.py

@@ -0,0 +1,27 @@
+"""The end-to-end time budget: one slow scenario fails it; a growing suite does not.
+
+Regression: a fixed 120 s total failed main at 185 scenarios, every one passing
+and none over 3 s. The budget was raised twice before for the same reason.
+"""
+
+from __future__ import annotations
+
+from tests.e2e.conftest import SUITE_SECONDS_PER_TEST, TEST_BUDGET_SECONDS, over_budget
+
+
+def test_a_large_suite_of_quick_scenarios_is_within_budget() -> None:
+    durations = {f"tests/e2e/test_x.py::test_{i}": 0.65 for i in range(185)}
+    assert over_budget(durations) == []
+
+
+def test_one_slow_scenario_is_named() -> None:
+    durations = {f"tests/e2e/test_x.py::test_{i}": 0.5 for i in range(20)}
+    durations["tests/e2e/test_x.py::test_slow"] = TEST_BUDGET_SECONDS + 1
+    [problem] = over_budget(durations)
+    assert "test_slow" in problem and "per-scenario" in problem
+
+
+def test_a_suite_whose_average_doubles_fails() -> None:
+    durations = {f"tests/e2e/test_x.py::test_{i}": SUITE_SECONDS_PER_TEST + 0.5 for i in range(50)}
+    [problem] = over_budget(durations)
+    assert "suite took" in problem