diff --git a/.github/scripts/run_dataset_acceptance.py b/.github/scripts/run_dataset_acceptance.py index 397a34f..7bbdce6 100644 --- a/.github/scripts/run_dataset_acceptance.py +++ b/.github/scripts/run_dataset_acceptance.py @@ -25,7 +25,13 @@ "tests/core/test_dataset_scale_acceptance.py::" "test_installed_default_export_short_final_shard_boundary" ) -REQUIRED_TEST_NODES = (TEST_NODE, BOUNDARY_TEST_NODE) +REQUIRED_TEST_NODES = ( + TEST_NODE, + BOUNDARY_TEST_NODE, + "tests/core/test_dataset_route_scale_acceptance.py::test_million_lifecycles_real_attach_and_scratch_routes", + "tests/core/test_dataset_route_scale_acceptance.py::test_million_lifecycles_over_limit_respects_disabled_scratch_budget", + "tests/core/test_dataset_route_scale_acceptance.py::test_million_lifecycles_real_subprocess_deep_offset_and_budget_precedence", +) class AcceptanceError(RuntimeError): diff --git a/tests/core/test_dataset_route_scale_acceptance.py b/tests/core/test_dataset_route_scale_acceptance.py new file mode 100644 index 0000000..aa0a6fa --- /dev/null +++ b/tests/core/test_dataset_route_scale_acceptance.py @@ -0,0 +1,282 @@ +"""Same-artifact real route and subprocess paging acceptance. + +Focused cases run on every PR. Slow cases use one million dynamic lifecycles +(2,999,998 real events), not one million events or an enormous capture. No test +asks SQLite to ATTACH beyond its actual limit. Signal/crash cleanup, concurrent +rewriters, ordered frames and producer acceptance are separate contracts. +""" + +import json +import sqlite3 +from pathlib import Path + +import pytest + +from pt_snap_cli.api import QueryLimits, SnapshotAnalyzer +from pt_snap_cli.core import dataset_lifecycle_scratch as scratch +from pt_snap_cli.core import dataset_lifecycle_sql as sql +from pt_snap_cli.core.dataset_lifecycle_connection import _AttachedSources +from pt_snap_cli.core.errors import QueryExecutionError +from tests.dataset_failure_support import ( + ENV_NAMES, + child_environment, + lifecycle_dataset, + run_cli, +) +from tests.test_dataset_focus import hashes + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + for name in ENV_NAMES: + monkeypatch.delenv(name, raising=False) + + +@pytest.fixture(scope="module") +def million_lifecycles(tmp_path_factory): + return lifecycle_dataset(tmp_path_factory.mktemp("million-lifecycles") / "dataset", 1_000_000) + + +def assert_source_unchanged(root, before): + assert hashes(root) == before + for suffix in ("-journal", "-wal", "-shm"): + assert not list(root.rglob(f"*{suffix}")) + + +def compare_routes(root, lifecycles, monkeypatch, record_property): + before = hashes(root) + assert sql._attachment_limit() >= 4, "Four-shard ATTACH evidence needs a legal runtime limit" + attached, built, connections, directories = [], [], [], [] + original_enter, original_build = _AttachedSources.__enter__, scratch._ScratchSources.build + + def enter(database): + result = original_enter(database) + names = [r[1] for r in database.connection.execute("PRAGMA database_list").fetchall()] + attached.append(names) + connections.append(database.connection) + return result + + def build(database, items, maximum): + result = original_build(database, items, maximum) + connection = database.connection + path = Path(connection.execute("PRAGMA database_list").fetchone()[2]) + directories.append(path.parent) + connections.append(connection) + built.append(connection.execute("SELECT COUNT(*) FROM lifecycle_state").fetchone()[0]) + assert path.stat().st_size <= maximum + return result + + monkeypatch.setattr(_AttachedSources, "__enter__", enter) + monkeypatch.setattr(scratch._ScratchSources, "build", build) + cases = [ + ("block", {"limit": 5, "offset": 2, "order_by": "id", "order_dir": "ASC"}), + ("leak_detection", {"limit": 5}), + ("freed_block_lifetime", {}), + ] + answers = [] + for threshold in (2 * lifecycles + 100, 0): + # Only the route decision changes. Actual SQL, storage and all default + # row/byte/scratch budgets are identical on the very same artifact. + monkeypatch.setattr(sql, "_ATTACHED_OBSERVATION_LIMIT", threshold) + with SnapshotAnalyzer(root) as analyzer: + answers.append( + [analyzer.execute_query(name, params, exact_total=True) for name, params in cases] + ) + assert attached == [["main", "s0", "s1", "s2", "s3"]] * 3 + assert built == [lifecycles + 1] * 3 + for first, second in zip(*answers, strict=True): + # Query counts are operational diagnostics, not semantic result fields. + for answer in (first, second): + answer["scope"]["source_coverage"].pop("source_queries") + assert first == second + block, leak, freed = answers[0] + assert block["total"] == lifecycles + 1 + assert [row["id"] for row in block["rows"]] == [3, 6, 9, 12, 15] + coverage = block["scope"]["source_coverage"] + assert coverage["lifecycles"] == lifecycles + 1 + assert coverage["allocation_events_resolved"] == lifecycles + assert coverage["free_events_resolved"] == lifecycles - 1 + assert coverage["unproved_identities"] == 0 + assert leak["total"] == 1 + assert leak["rows"][0]["allocEventId"] == 3 * (lifecycles - 1) + assert freed["rows"][0]["block_count"] == lifecycles - 1 + assert freed["rows"][0]["size_bytes"] == 16 * (lifecycles - 1) + assert freed["rows"][0]["lifetime_events"] == "<1k" + assert all(not path.exists() for path in directories) + for connection in connections: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + connection.execute("SELECT 1") + assert_source_unchanged(root, before) + record_property( + "route_acceptance", + json.dumps( + { + "dynamic_lifecycles": lifecycles, + "static_lifecycles": 1, + "events": 3 * lifecycles - 2, + "shards": 4, + "actual_attached_names": attached, + "scratch_lifecycle_counts": built, + "scratch_directories_removed": len(directories), + "source_hashes": before, + }, + default=str, + ), + ) + + +def active_and_crossing_contract(root, lifecycles): + before = hashes(root) + manifest = json.loads((root / "manifest.json").read_text()) + boundary = manifest["eventsPerSlice"] + allocation = boundary - boundary % 3 + if allocation == boundary: + allocation -= 3 + assert allocation < boundary <= allocation + 2 + with SnapshotAnalyzer(root) as analyzer: + row = analyzer.execute_query("block", {"allocEventId": allocation}, exact_total=True)[ + "rows" + ][0] + assert row["allocation_source"]["slice_index"] == 0 + assert row["free_source"]["slice_index"] == 1 + for template in ("active_blocks_at_event", "active_memory_callstack_at_event"): + for include_static, offset, total in ((True, 1, 2), (False, 0, 1)): + params = {"event_id": 3 * (lifecycles - 1), "include_static": include_static} + if template == "active_blocks_at_event": + params.update(offset=offset, limit=1) + else: + params["top_n"] = 1 + answer = analyzer.execute_query(template, params, exact_total=True) + assert answer["total"] == total + assert answer["returned"] == (1 if template == "active_blocks_at_event" else total) + assert answer["total_is_exact"] + if template == "active_blocks_at_event": + assert answer["rows"][0]["allocEventId"] == 3 * (lifecycles - 1) + else: + assert sorted(row["size_bytes"] for row in answer["rows"]) == ( + [16, 64] if include_static else [16] + ) + assert_source_unchanged(root, before) + + +def test_focused_real_routes_and_cross_shard_templates(tmp_path, monkeypatch, record_property): + root = lifecycle_dataset(tmp_path / "routes", 80) + compare_routes(root, 80, monkeypatch, record_property) + active_and_crossing_contract(root, 80) + + +@pytest.mark.nightly +@pytest.mark.slow +def test_million_lifecycles_real_attach_and_scratch_routes( + million_lifecycles, monkeypatch, record_property +): + compare_routes(million_lifecycles, 1_000_000, monkeypatch, record_property) + active_and_crossing_contract(million_lifecycles, 1_000_000) + + +def above_attachment_limit(root, lifecycles, monkeypatch): + before = hashes(root) + built, directories = [], [] + original = scratch._ScratchSources.build + + def build(database, items, maximum): + result = original(database, items, maximum) + built.append(len(items)) + path = Path(database.connection.execute("PRAGMA database_list").fetchone()[2]) + directories.append(path.parent) + return result + + def illegal_attach(database): + pytest.fail("Over-limit operation must select scratch or bounded Python fallback") + + monkeypatch.setattr(scratch._ScratchSources, "build", build) + monkeypatch.setattr(_AttachedSources, "__enter__", illegal_attach) + with SnapshotAnalyzer(root) as analyzer: + actual = analyzer.execute_query("block", {"limit": 5}, exact_total=True) + assert actual["total"] == lifecycles + 1 and actual["returned"] == 5 + assert actual["scope"]["source_coverage"]["allocation_events_resolved"] == lifecycles + limits = QueryLimits(max_scratch_db_bytes=0) + if lifecycles > limits.max_work_rows: + with pytest.raises(QueryExecutionError, match="work budget exceeded"): + analyzer.execute_query("block", {"limit": 5}, query_limits=limits) + else: + fallback = analyzer.execute_query( + "block", {"limit": 5}, exact_total=True, query_limits=limits + ) + assert fallback["rows"] == actual["rows"] + assert fallback["total"] == actual["total"] + assert built == [sql._attachment_limit() + 1] + assert all(not path.exists() for path in directories) + assert_source_unchanged(root, before) + + +def test_focused_over_limit_scratch_and_disabled_fallback(tmp_path, monkeypatch): + root = lifecycle_dataset(tmp_path / "over-limit", 80, sql._attachment_limit() + 1) + above_attachment_limit(root, 80, monkeypatch) + + +@pytest.mark.nightly +@pytest.mark.slow +def test_million_lifecycles_over_limit_respects_disabled_scratch_budget(tmp_path, monkeypatch): + root = lifecycle_dataset(tmp_path / "over-limit", 1_000_000, sql._attachment_limit() + 1) + above_attachment_limit(root, 1_000_000, monkeypatch) + + +def deep_cli_paging(root, lifecycles, tmp_path): + before = hashes(root) + offset = 100_001 if lifecycles == 1_000_000 else 30_000 + raised_budget = 500_000 if lifecycles == 1_000_000 else 300_000 + count = 3 * lifecycles - 2 + params = {"min_id": 0, "offset": offset, "limit": 5, "order_by": "address", "order_dir": "DESC"} + base = ["query", str(root), "--template-use", "event", "--exact-total", "--json"] + + def query(parameters, flags=(), env=None): + return run_cli(tmp_path, base + ["--params", json.dumps(parameters), *flags], env=env) + + failed = query(params) + assert failed.returncode == 1 and failed.stdout == "" + error = json.loads(failed.stderr)["error"] + assert error["code"] == "QUERY_FAILED" + assert "max_work_rows=100000" in error["message"] + assert "no partial global result" in error["message"] + raised = query(params, ["--max-work-rows", str(raised_budget)]) + by_id = query({**params, "order_by": "id"}) + env = child_environment(tmp_path) + env["PT_SNAP_MAX_WORK_ROWS"] = str(raised_budget) + from_env = query(params, env=env) + env["PT_SNAP_MAX_WORK_ROWS"] = "1" + overridden = query(params, ["--max-work-rows", str(raised_budget)], env=env) + env["PT_SNAP_MAX_WORK_ROWS"] = str(raised_budget) + restrictive_flag = query(params, ["--max-work-rows", "1"], env=env) + assert restrictive_flag.returncode == 1 and restrictive_flag.stdout == "" + assert "max_work_rows=1," in json.loads(restrictive_flag.stderr)["error"]["message"] + expected_ids = list(range(count - 1 - offset, count - 6 - offset, -1)) + for result, budget in ( + (raised, raised_budget), + (by_id, 100000), + (from_env, raised_budget), + (overridden, raised_budget), + ): + assert result.returncode == 0, result.stderr + assert result.stderr == "" + payload = json.loads(result.stdout) + assert [row["id"] for row in payload["rows"]] == expected_ids + assert payload["total"] == count and payload["returned"] == 5 + assert payload["total_is_exact"] and payload["has_more"] + assert payload["scope"]["work_budget"]["max_rows"] == budget + assert_source_unchanged(root, before) + + +def test_focused_real_subprocess_deep_offset_and_budget_precedence(tmp_path): + root = lifecycle_dataset(tmp_path / "paging", 40_000) + deep_cli_paging(root, 40_000, tmp_path) + + +@pytest.mark.nightly +@pytest.mark.slow +def test_million_lifecycles_real_subprocess_deep_offset_and_budget_precedence( + million_lifecycles, tmp_path +): + deep_cli_paging(million_lifecycles, 1_000_000, tmp_path) diff --git a/tests/dataset_failure_support.py b/tests/dataset_failure_support.py new file mode 100644 index 0000000..e6c215c --- /dev/null +++ b/tests/dataset_failure_support.py @@ -0,0 +1,253 @@ +"""Inspectable lifecycle workload and real-CLI fault harness for #252. + +This is synthetic SQL evidence, not producer/exporter or live-capture evidence. +One dynamic lifecycle has alloc/request/complete events; only the final one stays +live. A static carry-in is repeated in every shard. Membership and expected +answers are arithmetic, independent of production reducers and other fixtures. +""" + +import json +import os +import sqlite3 +import subprocess +import sys +from contextlib import ExitStack, closing, contextmanager +from pathlib import Path +from unittest.mock import patch + +REPO = Path(__file__).resolve().parents[1] +ENV_NAMES = ( + "PT_SNAP_DB_PATH", + "PT_SNAP_QUERY_TIMEOUT", + "PT_SNAP_MAX_WORK_ROWS", + "PT_SNAP_MAX_WORK_BYTES", + "PT_SNAP_MAX_SCRATCH_DB_BYTES", +) +SCHEMA = """ +CREATE TABLE dictionary ("table" TEXT,"column" TEXT,"key" TEXT,"value" TEXT); +CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY, action INTEGER, address INTEGER, + size INTEGER, stream INTEGER, allocated INTEGER, active INTEGER, reserved INTEGER, callstack TEXT); +CREATE TABLE block_0 (id INTEGER PRIMARY KEY, address INTEGER, size INTEGER, + requestedSize INTEGER, state INTEGER, allocEventId INTEGER, freeEventId INTEGER); +""" + + +def lifecycle_dataset(root, lifecycles, shards=4): + count = 3 * lifecycles - 2 + capacity = (count + shards - 1) // shards + # Make a real allocation/completion cross a boundary rather than arranging + # every shard at an already-completed lifetime boundary. + if capacity % 3 == 0: + capacity += 1 + assert (count + capacity - 1) // capacity == shards + (root / "device_0").mkdir(parents=True) + records = [] + for index, low in enumerate(range(0, count, capacity)): + high = min(count, low + capacity) - 1 + file = f"device_0/slice_{index:05d}.db" + with closing(sqlite3.connect(root / file)) as conn, conn: + conn.executescript(SCHEMA) + actions = ( + "segment_map", + "segment_unmap", + "segment_alloc", + "segment_free", + "alloc", + "free_requested", + "free_completed", + "workspace_snapshot", + ) + conn.executemany( + "INSERT INTO dictionary VALUES (?,?,?,?)", + [("trace_entry_0", "action", str(i), name) for i, name in enumerate(actions)] + + [ + ("block_0", "state", str(i), name) + for i, name in [ + (-1, "inactive"), + (0, "active_pending_free"), + (1, "active_allocated"), + ] + ], + ) + conn.executemany( + "INSERT INTO trace_entry_0 VALUES (?,?,?,?,?,?,?,?,?)", + ( + ( + i, + 4 + i % 3, + 1000, + 16, + 0, + 64 + 16 * (i % 3 == 0), + 64 + 16 * (i % 3 != 2), + 4096, + "allocate" if i % 3 == 0 else "free", + ) + for i in range(low, high + 1) + ), + ) + conn.execute( + "INSERT INTO trace_entry_0 VALUES (-1,2,0,4096,0,999999,999999,999999,'boundary')" + ) + conn.execute("INSERT INTO block_0 VALUES (-1,2000,64,64,1,-1,-1)") + conn.executemany( + "INSERT INTO block_0 VALUES (?,?,?,?,?,?,?)", + ( + (a, 1000, 16, 16, int(a + 1 >= low), a, a + 2 if a + 2 < count else -1) + for a in range(low - low % 3, high + 1, 3) + ), + ) + records.append( + {"index": index, "startEventId": low, "endEventId": high, "file": file, "ready": True} + ) + (root / "manifest.json").write_text( + json.dumps( + { + "schemaVersion": 1, + "status": "complete", + "sourceFile": "/synthetic/no-pickle", + "cacheHash": "", + "eventsPerSlice": capacity, + "devices": { + "0": { + "eventCount": count, + "sliceCount": shards, + "readySlices": list(range(shards)), + "slices": records, + } + }, + } + ) + ) + return root + + +def child_environment(tmp_path): + env = os.environ.copy() + for key in (*ENV_NAMES, "PYTHONHOME"): + env.pop(key, None) + env.update( + HOME=str(tmp_path), + USERPROFILE=str(tmp_path), + NO_COLOR="1", + TERM="dumb", + PYTHONPATH=os.pathsep.join((str(REPO / "src"), str(REPO))), + ) + return env + + +def run_cli(tmp_path, args, env=None, fault=None, evidence=None): + command = [ + sys.executable, + "-c", + "from pt_snap_cli.cli import _safe_call; raise SystemExit(_safe_call())", + ] + if fault is not None: + command = [sys.executable, "-m", "tests.dataset_failure_support", fault, str(evidence)] + return subprocess.run( + command + args, + cwd=tmp_path, + env=env or child_environment(tmp_path), + capture_output=True, + text=True, + timeout=180, + check=False, + ) + + +@contextmanager +def fault_hooks(kind, evidence): + """Fault real execution boundaries; never fabricate rows or query results.""" + from pt_snap_cli.core import dataset_lifecycle_scratch as scratch + from pt_snap_cli.core import dataset_lifecycle_sql as sql + from pt_snap_cli.core.dataset_lifecycle_connection import _AttachedSources + + connections, directories = [], [] + events = [] + original_enter = _AttachedSources.__enter__ + original_init = _AttachedSources.__init__ + original_execute = scratch._ScratchSources.execute + original_directory = scratch.tempfile.TemporaryDirectory + original_open = scratch.os.open + + def init(self, *args, **kwargs): + original_init(self, *args, **kwargs) + connections.append(self.connection) + + def enter(database): + if kind == "attach": + attempts = 0 + + def authorize(action, first, second, name, trigger): + nonlocal attempts + if action == sqlite3.SQLITE_ATTACH: + attempts += 1 + events.append(f"attach_attempt_{attempts}") + if attempts == 2: + return sqlite3.SQLITE_DENY + return sqlite3.SQLITE_OK + + database.connection.set_authorizer(authorize) + return original_enter(database) + + def directory(*args, **kwargs): + result = original_directory(*args, **kwargs) + directories.append(Path(result.name)) + events.append("scratch_created") + return result + + def open_file(path, *args, **kwargs): + if kind == "scratch_io" and Path(path).name == "derived.db": + events.append("scratch_open_fault") + raise OSError("injected scratch open failure") + return original_open(path, *args, **kwargs) + + def execute(database, statement, values=()): + result = original_execute(database, statement, values) + if kind == "timeout" and statement.startswith("INSERT INTO lifecycle_state"): + events.append("reduction_insert_completed") + # Expire the real operation deadline after real scratch work. No + # sleeps, mocked error object, or replacement query result is needed. + database.sources.budget.started -= database.sources.budget.timeout_s + 1 + return result + + with ExitStack() as patches: + for owner, name, replacement in ( + (_AttachedSources, "__init__", init), + (_AttachedSources, "__enter__", enter), + (scratch._ScratchSources, "execute", execute), + (scratch.tempfile, "TemporaryDirectory", directory), + (scratch.os, "open", open_file), + ): + patches.enter_context(patch.object(owner, name, replacement)) + if kind in ("timeout", "scratch_io", "scratch_space"): + patches.enter_context(patch.object(sql, "_ATTACHED_OBSERVATION_LIMIT", 0)) + try: + yield + finally: + closed = [] + for connection in connections: + try: + connection.execute("SELECT 1").close() + except sqlite3.ProgrammingError: + closed.append(True) + else: + closed.append(False) + evidence.update( + events=events, + connections_closed=closed, + scratch_removed=[not p.exists() for p in directories], + ) + + +if __name__ == "__main__": + fault, destination = sys.argv[1:3] + sys.argv = ["pt-snap", *sys.argv[3:]] + observation = {} + try: + with fault_hooks(fault, observation): + from pt_snap_cli.cli import _safe_call + + raise SystemExit(_safe_call()) + finally: + Path(destination).write_text(json.dumps(observation)) diff --git a/tests/test_dataset_acceptance_runner.py b/tests/test_dataset_acceptance_runner.py index b409185..15f1d3c 100644 --- a/tests/test_dataset_acceptance_runner.py +++ b/tests/test_dataset_acceptance_runner.py @@ -3,6 +3,7 @@ import hashlib import importlib.util import json +import re import subprocess import sys import xml.etree.ElementTree as ET @@ -12,6 +13,11 @@ RUNNER = Path(".github/scripts/run_dataset_acceptance.py") BOUNDARY_NAME = "test_installed_default_export_short_final_shard_boundary" +SCALE_TEST_NODES = ( + "tests/core/test_dataset_route_scale_acceptance.py::test_million_lifecycles_real_attach_and_scratch_routes", + "tests/core/test_dataset_route_scale_acceptance.py::test_million_lifecycles_over_limit_respects_disabled_scratch_budget", + "tests/core/test_dataset_route_scale_acceptance.py::test_million_lifecycles_real_subprocess_deep_offset_and_budget_precedence", +) @pytest.fixture @@ -45,6 +51,15 @@ def write_report(path, name, evidence, outcome=None, *, include_boundary=True): file="tests/core/test_dataset_scale_acceptance.py", classname="tests.core.test_dataset_scale_acceptance", ) + for node in SCALE_TEST_NODES: + file, name = node.split("::") + ET.SubElement( + suite, + "testcase", + name=name, + file=file, + classname=file.removesuffix(".py").replace("/", "."), + ) ET.ElementTree(suite).write(path) @@ -119,6 +134,7 @@ def test_runner_accepts_additional_nightly_cases_and_records_complete_identities assert [runner.test_node_id(case) for case in runner.read_cases(report)] == [ runner.TEST_NODE, runner.BOUNDARY_TEST_NODE, + *SCALE_TEST_NODES, "tests/test_extra.py::TestRoutes::test_extra[v2]", ] @@ -302,3 +318,29 @@ def test_runner_rejects_nonpassing_short_final_shard_case(runner, tmp_path, outc ) with pytest.raises(runner.AcceptanceError, match="without skips"): runner.read_evidence(report, "abc") + + +def test_required_nightly_registry_contains_actual_scale_gates(runner): + assert runner.REQUIRED_TEST_NODES == ( + runner.TEST_NODE, + runner.BOUNDARY_TEST_NODE, + *SCALE_TEST_NODES, + ) + + +@pytest.mark.parametrize("node", SCALE_TEST_NODES) +@pytest.mark.parametrize("outcome", ["missing", "skipped", "failure", "error"]) +def test_every_scale_gate_is_required_by_full_identity(runner, tmp_path, node, outcome): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + tree = ET.parse(report) + case = next(case for case in tree.findall(".//testcase") if runner.test_node_id(case) == node) + if outcome == "missing": + # A same-named passing function in another module cannot replace a gate. + case.set("file", "tests/test_extra.py") + case.set("classname", "tests.test_extra") + else: + ET.SubElement(case, outcome) + tree.write(report) + with pytest.raises(runner.AcceptanceError, match=re.escape(node)): + runner.read_evidence(report, "abc") diff --git a/tests/test_dataset_failure_acceptance.py b/tests/test_dataset_failure_acceptance.py new file mode 100644 index 0000000..ccbfda1 --- /dev/null +++ b/tests/test_dataset_failure_acceptance.py @@ -0,0 +1,111 @@ +"""Real subprocess adapter parity for bounded failures, without partial output.""" + +import json + +import pytest + +from pt_snap_cli.api import QueryLimits, SnapshotAnalyzer +from pt_snap_cli.core.errors import QueryExecutionError, QueryTimeoutError +from tests.dataset_failure_support import ( + ENV_NAMES, + fault_hooks, + lifecycle_dataset, + run_cli, +) +from tests.test_dataset_focus import hashes + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + for name in ENV_NAMES: + monkeypatch.delenv(name, raising=False) + + +@pytest.mark.parametrize( + "kind", ["rows", "bytes", "timeout", "attach", "scratch_space", "scratch_io"] +) +def test_real_cli_text_json_and_api_preserve_typed_failure_and_cleanup(tmp_path, kind): + root = lifecycle_dataset(tmp_path / "dataset", 40) + before = hashes(root) + limits = QueryLimits( + max_work_rows=1 if kind == "rows" else 100_000, + max_work_bytes=1 if kind == "bytes" else 64 * 1024**2, + max_scratch_db_bytes=( + 4096 if kind == "scratch_space" else 0 if kind == "attach" else 512 * 1024**2 + ), + ) + expected_type = QueryTimeoutError if kind == "timeout" else QueryExecutionError + api_evidence = {} + with fault_hooks(kind, api_evidence), SnapshotAnalyzer(root) as analyzer: + with pytest.raises(expected_type) as caught: + analyzer.execute_query("block", {"limit": 5}, timeout_s=30.0, query_limits=limits) + assert type(caught.value) is expected_type + message = str(caught.value) + assert message.count("Query execution failed:") <= 1 + assert not message.startswith("Query execution failed:") + assert { + "rows": "work budget exceeded", + "bytes": "work budget exceeded", + "timeout": "timed out after 30.0 seconds", + "attach": "not authorized", + "scratch_space": "scratch space exhausted", + "scratch_io": "scratch storage failed", + }[kind] in message + observations = [api_evidence] + for json_mode in (False, True): + evidence = tmp_path / f"evidence-{json_mode}.json" + args = [ + "query", + str(root), + "--template-use", + "block", + "--params", + '{"limit":5}', + "--timeout", + "30", + "--max-work-rows", + str(limits.max_work_rows), + "--max-work-bytes", + str(limits.max_work_bytes), + "--max-scratch-db-bytes", + str(limits.max_scratch_db_bytes), + ] + result = run_cli( + tmp_path, args + (["--json"] if json_mode else []), fault=kind, evidence=evidence + ) + assert result.returncode == 1, (result.stdout, result.stderr) + assert "Traceback" not in result.stdout + result.stderr + if json_mode: + assert result.stdout == "" + payload = json.loads(result.stderr) + assert payload["error"]["message"] == message + assert payload["error"]["code"] == ( + "QUERY_TIMEOUT" if kind == "timeout" else "QUERY_FAILED" + ) + assert not {"rows", "total", "returned", "scope"} & payload.keys() + else: + assert result.stderr == "" + assert result.stdout == f"Error executing query: {message}\n" + observations.append(json.loads(evidence.read_text())) + assert hashes(root) == before + for evidence in observations: + assert all(evidence["connections_closed"]) + assert all(evidence["scratch_removed"]) + if kind == "attach": + assert evidence["events"] == ["attach_attempt_1", "attach_attempt_2"] + assert evidence["connections_closed"] == [True] + if kind.startswith("scratch") or kind == "timeout": + assert evidence["scratch_removed"] == [True] + assert "scratch_created" in evidence["events"] + if kind == "timeout": + assert "reduction_insert_completed" in evidence["events"] + # Faults do not poison admission/cache state or prevent an ordinary retry. + with SnapshotAnalyzer(root) as analyzer: + result = analyzer.execute_query("block", {"limit": 5}, exact_total=True) + assert result["total"] == 41 and result["returned"] == 5 + assert hashes(root) == before + assert not any(root.rglob("*-journal")) + assert not any(root.rglob("*-wal")) + assert not any(root.rglob("*-shm"))