From f444ffea9364905f19fb57e5057867d18ea5d220 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 14:49:03 +0000 Subject: [PATCH 01/32] WIP: correct native slot identity, snapshots and initialization writes --- conformance/runtime_provider_adapter.py | 189 ++++++++++++++++++++---- src/determa/state/definition.py | 13 +- src/determa/state/engine.py | 20 ++- src/determa/state/inspection.py | 10 +- src/determa/state/queueing.py | 1 + src/determa/state/runtime_providers.py | 69 +++++---- tests/test_runtime_providers.py | 8 +- 7 files changed, 236 insertions(+), 74 deletions(-) diff --git a/conformance/runtime_provider_adapter.py b/conformance/runtime_provider_adapter.py index 5d8cf90..287df36 100644 --- a/conformance/runtime_provider_adapter.py +++ b/conformance/runtime_provider_adapter.py @@ -26,6 +26,7 @@ RuntimeProviderError, RuntimeProviderRegistry, SourceClosure, + ValidationError, admit, compile_language_source, create, @@ -112,19 +113,11 @@ def _installed( ) descriptors: list[dict[str, Any]] = [] - def scan(value: Any) -> None: - if type(value) is dict: - if type(value.get("guard")) is dict and "provider" in value["guard"]: - descriptors.append({"kind": "guard", "binding": value["guard"]["provider"]}) - if "provider_actions" in value: - descriptors.append({"kind": "actions", "binding": value["provider_actions"]}) - for child in value.values(): - scan(child) - elif type(value) is list: - for child in value: - scan(child) - - scan(bundle_document) + from determa.state.runtime_providers import _runtime_bindings + + descriptors = [ + {"kind": kind, "binding": binding} for kind, binding in _runtime_bindings(bundle_document) + ] names = ( {"example.native-common", "example.guard-compiler"} if compiler @@ -187,6 +180,8 @@ def scan(value: Any) -> None: "invalid": bool(arguments.get("invalid_output", False)), "fail": bool(arguments.get("action_fail", False)), "external_io": bool(arguments.get("action_external_io", False)), + **({"repeat_send": True} if arguments.get("repeat_send") else {}), + **({"mixed_send": True} if arguments.get("mixed_send") else {}), } ), ) @@ -208,7 +203,11 @@ def _state(aggregate: dict[str, Any]) -> dict[str, Any]: item for item in aggregate["runtimes"] if item["runtime_id"] == aggregate["root_runtime_id"] ) return { - "active_leaf": runtime["active_leaf_state_definition_pointers"][-1], + "active_leaf": ( + runtime["active_leaf_state_definition_pointers"][-1] + if runtime["active_leaf_state_definition_pointers"] + else None + ), "status": runtime["status"], "variables": { item["variable_declaration_pointer"].split("/")[-1]: item["value"] @@ -251,7 +250,14 @@ def _run_step( observation["stages"].append("admit") observation["state_before"] = _state(aggregate) observation["stages"].append("evaluate_cel") - result = step(aggregate, setup["target_runtime_id"], resolver) + result = step( + aggregate, + setup["target_runtime_id"], + resolver, + _include_host_evidence=bool( + request["arguments"].get("repeat_send") or request["arguments"].get("mixed_send") + ), + ) _provider_counts(registry, observation) if observation["calls"]["guard"]: observation["stages"].append("evaluate_guard") @@ -263,8 +269,12 @@ def _run_step( observation["result"] = "faulted" observation["code"] = result["fault"]["code"] observation["state_after"] = observation["state_before"] - if request["arguments"].get("invalid_output"): + if request["arguments"].get("invalid_output") or request["arguments"].get( + "destroyed_write" + ): observation["value"] = {"boundary_code": "runtime_provider_output_invalid"} + if request["arguments"].get("destroyed_write"): + observation["value"]["source_locator"] = result["fault"]["source_locator"] return observation["stages"].append("commit") observation["result"] = ( @@ -272,10 +282,68 @@ def _run_step( ) observation["determa_state_committed"] = True observation["state_after"] = _state(result["state"]) - observation["value"] = { - "accepted": observation["state_after"]["variables"]["accepted"], - "emissions": len(result["emissions"]), - } + observation["value"] = {"emissions": len(result["emissions"])} + if "accepted" in observation["state_after"]["variables"]: + observation["value"]["accepted"] = observation["state_after"]["variables"]["accepted"] + if observation["state_after"]["status"] == "completed": + observation["value"].update( + status="completed", exit_correlation=result["emissions"][-1]["correlation_id"] + ) + if request["arguments"].get("repeat_send") or request["arguments"].get("mixed_send"): + observation["value"]["emission_identities"] = [ + ( + { + "event_id": item["event_id"], + "emission_index": item["emission_index"], + "acceptance_sequence": item["acceptance_sequence"], + "queue_sequence": item["queue_sequence"], + } + if "kind" in item + else { + "effect_id": item["effect_id"], + "sequence": item["sequence"], + "emission_index": item["_determa_v1_emission_index"], + } + ) + for item in result["emissions"] + ] + + if request["arguments"].get("capture_snapshot"): + for active in registry._active.values(): + for key in ("guard_snapshot", "action_snapshot"): + captured = getattr(active.provider, key, None) + if captured is not None: + observation["value"][key] = captured + + +def _run_create( + bundle: Any, + registry: RuntimeProviderRegistry, + request: dict[str, Any], + observation: dict[str, Any], +) -> None: + creation = request["setup"]["create_request"] + result = create( + bundle, + creation["machine_id"], + creation["root_instance_id"], + creation["creation_id"], + creation["bindings"], + ) + observation["stages"].append("create") + _provider_counts(registry, observation) + if observation["calls"]["actions"]: + observation["stages"].extend(("evaluate_actions", "validate_output")) + observation["state_after"] = _state(result["state"]) + observation["result"] = result["status"] + observation["code"] = result["fault"]["code"] if result["fault"] else None + if result["fault"]: + observation["value"] = { + "boundary_code": "runtime_provider_output_invalid", + "source_locator": result["fault"]["source_locator"], + "emissions": len(result["emissions"]), + "status": result["status"], + } class _ConflictStore(MemoryExecutionStore): @@ -356,17 +424,17 @@ def _run_host_commit( entry = runtime["ready_mailbox"][0] store.conflict = request["arguments"]["cas_conflict"] store.replace_attempts = 0 + process_arguments = { + "expected_revision": checkpoint["revision"], + "expected_checkpoint_digest": checkpoint["execution_checkpoint_digest"], + "event_id": entry["envelope"]["event_id"], + "envelope_digest": entry["envelope_digest"], + "acceptance_sequence": entry["acceptance_sequence"], + "queue_sequence": entry["queue_sequence"], + } try: - host.process_ready_v1( - root_id, - setup["target_runtime_id"], - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - event_id=entry["envelope"]["event_id"], - envelope_digest=entry["envelope_digest"], - acceptance_sequence=entry["acceptance_sequence"], - queue_sequence=entry["queue_sequence"], - ) + first = host.process_ready_v1(root_id, setup["target_runtime_id"], **process_arguments) + except ExecutionHostError as exc: _provider_counts(registry, observation) if observation["calls"]["guard"]: @@ -381,7 +449,58 @@ def _run_host_commit( after = host.read_checkpoint(root_id) assert after is not None and after.source_bytes == admitted.source_bytes return - raise AssertionError("expected one real store compare-and-swap conflict") + if request["arguments"]["cas_conflict"]: + raise AssertionError("expected one real store compare-and-swap conflict") + _provider_counts(registry, observation) + if observation["calls"]["guard"]: + observation["stages"].append("evaluate_guard") + if observation["calls"]["actions"]: + observation["stages"].extend(("evaluate_actions", "validate_output")) + observation["stages"].extend(("compare_and_swap", "commit")) + committed = host.read_checkpoint(root_id) + assert committed is not None and store.replace_attempts == 1 + observation["state_after"] = _state(committed.document["root_record"]["aggregate_state"]) + receipt = first["receipt"] + references = receipt["emission_references"] + intents = committed.document["pending_outbox_intents"] + observation["value"] = { + "accepted": observation["state_after"]["variables"]["accepted"], + "emissions": len(first["core_result"]["emissions"]), + "emission_identities": [ + { + "effect_id": item["effect_id"], + "emission_index": next( + reference["emission_index"] + for reference in references + if reference.get("effect_id") == item["effect_id"] + ), + "sequence": item["sequence"], + } + for item in first["core_result"]["emissions"] + ], + "checkpoint_revision": committed.document["revision"], + "retained_effect_references": copy.deepcopy(references), + "pending_outbox_entries": copy.deepcopy(intents), + } + providers_before = [ + copy.deepcopy(active.provider.__dict__) for active in registry._active.values() + ] + replay = host.process_ready_v1(root_id, setup["target_runtime_id"], **process_arguments) + after_replay = host.read_checkpoint(root_id) + assert after_replay is not None + observation["value"].update( + replay_receipt_equal=replay == receipt, + replay_checkpoint_unchanged=after_replay.source_bytes == committed.source_bytes, + replay_provider_calls_unchanged=providers_before + == [active.provider.__dict__ for active in registry._active.values()], + ) + observation["stages"].append("replay") + observation["result"] = ( + "handled_now" + if receipt["outcome"]["disposition"] == "handled" + else receipt["outcome"]["disposition"] + ) + observation["determa_state_committed"] = True def _run_inspect( @@ -565,6 +684,8 @@ def execute(payload: dict[str, Any]) -> dict[str, Any]: observation["result"] = "accepted" elif operation == "step": _run_step(bundle, registry, request, observation) + elif operation == "create": + _run_create(bundle, registry, request, observation) elif operation == "host_commit": _run_host_commit(bundle, registry, request, observation) elif operation == "inspect": @@ -583,6 +704,12 @@ def execute(payload: dict[str, Any]) -> dict[str, Any]: observation["effective_capabilities"] = _profile() else: _provider_counts(registry, observation) + except ValidationError as exc: + # Static validation precedes provider capability verification inside the loader. + if observation["stages"][-1:] == ["verify_capabilities"]: + observation["stages"].pop() + observation["stages"].append("load") + observation["code"] = exc.code except RuntimeProviderError as exc: observation["code"] = exc.code return { diff --git a/src/determa/state/definition.py b/src/determa/state/definition.py index d16cf6a..5b63565 100644 --- a/src/determa/state/definition.py +++ b/src/determa/state/definition.py @@ -238,13 +238,6 @@ def load_bundle( def _has_runtime_provider(value: Any) -> bool: - if isinstance(value, dict): - return ( - "provider_actions" in value - or isinstance(value.get("guard"), dict) - and "provider" in value["guard"] - or any(_has_runtime_provider(child) for child in value.values()) - ) - if isinstance(value, list): - return any(_has_runtime_provider(child) for child in value) - return False + from .runtime_providers import _runtime_bindings + + return next(_runtime_bindings(value), None) is not None diff --git a/src/determa/state/engine.py b/src/determa/state/engine.py index f8ecbbe..cb561f7 100644 --- a/src/determa/state/engine.py +++ b/src/determa/state/engine.py @@ -392,6 +392,7 @@ def _dispatch( delivery: _Delivery = None, *, _capture_emission_provenance: bool = False, + _provider_envelope: dict[str, Any] | None = None, ) -> _Result: """Validate and process at most one envelope against an aggregate copy.""" validated = _coerce_bundle(bundle) @@ -458,6 +459,9 @@ def _dispatch( assert declaration is not None normalized_envelope["payload"] = _normalize_payload(declaration, envelope.get("payload")) execution.event = normalized_envelope + if _provider_envelope is not None: + execution.provider_event = copy.deepcopy(_provider_envelope) + execution.provider_event["payload"] = copy.deepcopy(normalized_envelope["payload"]) execution.cause_id = str(envelope["event_id"]) before = copy.deepcopy(state) try: @@ -1498,6 +1502,7 @@ class _Execution: emissions: list[dict[str, Any]] cause_id: str event: dict[str, Any] | None + provider_event: dict[str, Any] | None capture_emission_provenance: bool def __init__( @@ -1516,6 +1521,7 @@ def __init__( self.emissions = [] self.cause_id = "" self.event = None + self.provider_event = None self.capture_emission_provenance = capture_emission_provenance self._provider_writes: list[tuple[str, str]] = [] @@ -1686,6 +1692,7 @@ def enter_state( return if state.type == "composite" and descend: initial = state.raw["initial"] + write_start = len(self._provider_writes) target, history = self.resolve_compound_transition( runtime, machine, @@ -1695,6 +1702,7 @@ def enter_state( event_visible=False, ) assert target is not None + self.validate_provider_writes(runtime, machine, state, target, False, write_start) self.enter_path(runtime, machine, state, target, history=history) def initialize_variables( @@ -1788,7 +1796,8 @@ def provider_snapshot( ) -> dict[str, Any]: from .runtime_providers import guard_snapshot - return guard_snapshot(activation, self.event, binding) + event = (self.provider_event or self.event) if "event" in activation else None + return guard_snapshot(activation, event, binding) def allocate_components( self, runtime: dict[str, Any], machine: MachineModel, state: StateNode @@ -2143,6 +2152,7 @@ def run_provider_proposals( from .runtime_providers import RuntimeProviderError from .wire import decoded_typed_value + ordinals = {"internal": 0, "external": 0} for proposal in proposals: try: if "assign" in proposal: @@ -2165,6 +2175,7 @@ def run_provider_proposals( pointer, event_visible=self.event is not None, literal_values=True, + native_ordinals=ordinals, ) except (DetermaError, ValueError, KeyError, TypeError) as exc: raise RuntimeProviderError("runtime_provider_output_invalid") from exc @@ -2179,6 +2190,7 @@ def send( *, event_visible: bool, literal_values: bool = False, + native_ordinals: dict[str, int] | None = None, ) -> None: activation = self.activation(runtime, machine, state, event_visible=event_visible) declaration = self.event_declaration(runtime, send["event"]) @@ -2262,6 +2274,10 @@ def send( ): raise StepFault(FaultCode.ACTION_FAULT, pointer) for index, target in enumerate(resolved): + if native_ordinals is not None: + kind = "external" if target == "external" else "internal" + index = native_ordinals[kind] + native_ordinals[kind] += 1 if target == "external": sequence = int(self.state["next_output_sequence"]) self.state["next_output_sequence"] = sequence + 1 @@ -2570,6 +2586,7 @@ def descend_composite( self.enter_path(runtime, machine, state, destination, history=False) return initial = state.raw["initial"] + write_start = len(self._provider_writes) target, target_history = self.resolve_compound_transition( runtime, machine, @@ -2579,6 +2596,7 @@ def descend_composite( event_visible=False, ) assert target is not None + self.validate_provider_writes(runtime, machine, state, target, False, write_start) self.enter_path(runtime, machine, state, target, history=target_history) def exit_state(self, runtime: dict[str, Any], machine: MachineModel, state: StateNode) -> None: diff --git a/src/determa/state/inspection.py b/src/determa/state/inspection.py index 8c87e66..57ae6da 100644 --- a/src/determa/state/inspection.py +++ b/src/determa/state/inspection.py @@ -245,7 +245,15 @@ def inspect_candidate( from .runtime_providers import RuntimeProviderError, guard_snapshot binding = guard["provider"] - snapshot = guard_snapshot(activation, candidate, binding) + snapshot = guard_snapshot( + activation, + { + **candidate, + "source": copy.deepcopy(envelope["source"]), + "cause_id": envelope["cause_id"], + }, + binding, + ) try: value, _charged, cost = runtime_providers.inspect_guard( binding, snapshot, guard_count + 1, remaining diff --git a/src/determa/state/queueing.py b/src/determa/state/queueing.py index 0fac8ce..18b6a92 100644 --- a/src/determa/state/queueing.py +++ b/src/determa/state/queueing.py @@ -883,6 +883,7 @@ def step_aggregate_v1( restored.state, {selected["delivery_mode"]: native}, _capture_emission_provenance=True, + _provider_envelope=selected["envelope"], ) disposition = result["disposition"] if disposition == DispositionCode.REJECTED.value: diff --git a/src/determa/state/runtime_providers.py b/src/determa/state/runtime_providers.py index f458224..9adec2b 100644 --- a/src/determa/state/runtime_providers.py +++ b/src/determa/state/runtime_providers.py @@ -14,13 +14,50 @@ import json import sys import types -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass from pathlib import Path from typing import Any from .extensions import ExtensionError, ExtensionRegistry + +def _runtime_bindings( + definition: Mapping[str, Any], +) -> Iterator[tuple[str, Mapping[str, Any]]]: + """Visit only executable grammar slots; metadata and values remain inert.""" + + def actions(items: Any) -> Iterator[tuple[str, Mapping[str, Any]]]: + for action in items or []: + if "provider_actions" in action: + yield "actions", action["provider_actions"] + + def transitions(value: Any) -> Iterator[tuple[str, Mapping[str, Any]]]: + for transition in value if isinstance(value, list) else [value]: + if not transition: + continue + guard = transition.get("guard") + if isinstance(guard, Mapping) and "provider" in guard: + yield "guard", guard["provider"] + yield from actions(transition.get("action")) + + def state(value: Mapping[str, Any]) -> Iterator[tuple[str, Mapping[str, Any]]]: + yield from actions(value.get("entry")) + yield from actions(value.get("exit")) + yield from transitions(value.get("initial")) + yield from transitions(value.get("choice")) + for handler in (value.get("on_events") or {}).values(): + yield from transitions(handler) + for child in (value.get("states") or {}).values(): + yield from state(child) + for component in value.get("components") or []: + if "root" in component: + yield from state(component["root"]) + + for machine in definition.get("machines") or []: + yield from state(machine["root"]) + + SourceIdentityVerifier = Callable[[Path, Any], bool] @@ -1079,19 +1116,8 @@ def resolve(self, kind: str, binding: Mapping[str, Any]) -> ResolvedRuntimeProvi def resolve_definition(self, definition: Mapping[str, Any]) -> None: """Preflight every executable slot, including unreachable declarations.""" - def visit(value: Any) -> None: - if isinstance(value, dict): - if isinstance(value.get("guard"), dict) and "provider" in value["guard"]: - self.resolve("guard", value["guard"]["provider"]) - if "provider_actions" in value: - self.resolve("actions", value["provider_actions"]) - for child in value.values(): - visit(child) - elif isinstance(value, list): - for child in value: - visit(child) - - visit(definition) + for kind, binding in _runtime_bindings(definition): + self.resolve(kind, binding) def effective_capabilities(self, definition: Mapping[str, Any]) -> dict[str, bool]: """Compose verified provider guarantees for one complete definition.""" @@ -1105,18 +1131,6 @@ def effective_capabilities(self, definition: Mapping[str, Any]) -> dict[str, boo report = dict.fromkeys(names, True) report["external_io_capable"] = False - def visit(value: Any) -> None: - if isinstance(value, dict): - if isinstance(value.get("guard"), dict) and "provider" in value["guard"]: - combine(self.resolve("guard", value["guard"]["provider"]).capabilities) - if "provider_actions" in value: - combine(self.resolve("actions", value["provider_actions"]).capabilities) - for child in value.values(): - visit(child) - elif isinstance(value, list): - for child in value: - visit(child) - def combine(claims: Mapping[str, bool]) -> None: for name in names: report[name] = report[name] and claims.get(name) is True @@ -1124,7 +1138,8 @@ def combine(claims: Mapping[str, bool]) -> None: report["external_io_capable"] or claims.get("external_io_capable") is True ) - visit(definition) + for kind, binding in _runtime_bindings(definition): + combine(self.resolve(kind, binding).capabilities) return report def invoke_guard(self, binding: Mapping[str, Any], snapshot: Mapping[str, Any]) -> bool: diff --git a/tests/test_runtime_providers.py b/tests/test_runtime_providers.py index 2e50244..533a5c4 100644 --- a/tests/test_runtime_providers.py +++ b/tests/test_runtime_providers.py @@ -409,13 +409,13 @@ def test_semantic_and_ordinary_guards_see_same_native_event(tmp_path: Path) -> N source_path.read_text() .replace( "self.ordinary_calls += 1\n return True", - 'self.ordinary_calls += 1\n return "source" not in snapshot["event"] ' - 'and "cause_id" not in snapshot["event"]', + 'self.ordinary_calls += 1\n return snapshot["event"]["source"]["host"] is True ' + 'and snapshot["event"]["cause_id"] == snapshot["event"]["event_id"]', ) .replace( "return True, 1, 2", - 'return "source" not in snapshot["event"] ' - 'and "cause_id" not in snapshot["event"], 1, 2', + 'return snapshot["event"]["source"]["host"] is True ' + 'and snapshot["event"]["cause_id"] == snapshot["event"]["event_id"], 1, 2', ) ) source_path.write_text(source) From c0c51d65b19e7717a5464efafd0b29e5e9d535f8 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 14:55:21 +0000 Subject: [PATCH 02/32] WIP: pin corrected native conformance and refresh only version-1 schemas --- AGENTS.md | 6 ++--- README.md | 6 ++--- conformance/pins.py | 4 ++-- conformance/test_conformance.py | 2 +- scripts/sync_schema.py | 39 ++++++++++++++++++++++++++++----- 5 files changed, 42 insertions(+), 15 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index cacab48..61b5cd9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,11 +11,11 @@ package so it can coexist with the umbrella `determa` launcher. The implementation is conformant only when it passes the language-neutral suite. The implementation uses these immutable inputs: -- specification: `86bb88dd21cb1f799eefe5020b6e49dabf6e7225`; -- conformance: `affe3fe3bcc4d13fa7c5374471568e94af36f0d1` (99 format-1 +- specification: `77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0`; +- conformance: `cf7e673b26ebb162dfdc54d9d235300cd3350438` (99 format-1 core cases, 162 version-1 vectors, 142 durable-host vectors, 49 core inspection vectors, 47 extension negotiation vectors, 7 optional native-provider vectors, - 30 runtime-provider vectors, and 484 JSON artifacts). + 41 runtime-provider vectors, and 487 JSON artifacts). The package metadata is `0.3.0`. diff --git a/README.md b/README.md index fa3e486..5018d07 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,11 @@ Python implementation of [Determa State](https://github.com/fruwehq/determa-stat a language-agnostic statechart engine with a shared normative conformance suite. This implementation supports Determa State `format: 1` at specification -commit `86bb88dd21cb1f799eefe5020b6e49dabf6e7225`. Correctness is determined by +commit `77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0`. Correctness is determined by 99 format-1 core cases, 162 version-1 vectors, 142 durable-host vectors, 49 core inspection vectors, 47 extension negotiation vectors, 7 native guard-provider -inspection vectors, 30 runtime-provider vectors, and 484 manifest artifacts at -conformance commit `affe3fe3bcc4d13fa7c5374471568e94af36f0d1`. The native +inspection vectors, 41 runtime-provider vectors, and 487 manifest artifacts at +conformance commit `cf7e673b26ebb162dfdc54d9d235300cd3350438`. The native runtime-provider profile is optional and is exercised through its production adapter. The package metadata is `0.3.0`. Artifact and checkpoint schema version 1 is the diff --git a/conformance/pins.py b/conformance/pins.py index c083a0c..e5e259c 100644 --- a/conformance/pins.py +++ b/conformance/pins.py @@ -4,8 +4,8 @@ from pathlib import Path -CONFORMANCE_COMMIT = "affe3fe3bcc4d13fa7c5374471568e94af36f0d1" -SPEC_COMMIT = "86bb88dd21cb1f799eefe5020b6e49dabf6e7225" +CONFORMANCE_COMMIT = "cf7e673b26ebb162dfdc54d9d235300cd3350438" +SPEC_COMMIT = "77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0" ROOT = Path(__file__).resolve().parent.parent CONFORMANCE_CACHE = ROOT / ".cache" / f"determa-state-conformance-{CONFORMANCE_COMMIT[:12]}" diff --git a/conformance/test_conformance.py b/conformance/test_conformance.py index ddbed0b..c648fa0 100644 --- a/conformance/test_conformance.py +++ b/conformance/test_conformance.py @@ -148,7 +148,7 @@ def test_suite_present() -> None: assert len(core_cases()) == 99 assert len(version1_vectors()) == 162 assert len(durable_host_vectors()) == 142 - assert len(_manifest_artifacts()) == 484 + assert len(_manifest_artifacts()) == 487 @pytest.mark.parametrize("stored", [b"mutated", None]) diff --git a/scripts/sync_schema.py b/scripts/sync_schema.py index 0374681..28250cb 100644 --- a/scripts/sync_schema.py +++ b/scripts/sync_schema.py @@ -20,14 +20,41 @@ ROOT = Path(__file__).resolve().parent.parent DEST = ROOT / "src" / "determa" / "state" / "data" SCHEMAS = ( + "aggregate-state-package-v1.schema.json", + "aggregate-state-v1.schema.json", + "archive-export-request-v1.schema.json", + "archive-import-request-v1.schema.json", + "archive-participant-v1.schema.json", + "archive-result-v1.schema.json", + "archive-v1.schema.json", + "compilation-manifest-v1.schema.json", + "core-step-result-v1.schema.json", + "effect-cancellation-request-v1.schema.json", + "effect-cancellation-response-v1.schema.json", + "effect-result-request-v1.schema.json", + "effect-result-response-v1.schema.json", + "execution-checkpoint-v1.schema.json", + "extension-capability-report-v1.schema.json", + "extension-capability-requirement-v1.schema.json", + "extension-descriptor-v1.schema.json", + "host-authority-operation-v1.schema.json", + "host-authority-profile-report-v1.schema.json", + "host-effect-claim-v1.schema.json", + "host-effect-journal-v1.schema.json", + "inspection-v1.schema.json", + "language-source-v1.schema.json", "machine.schema.json", - "aggregate-state-v2.schema.json", - "migration-descriptor-v2.schema.json", - "aggregate-state-package-v2.schema.json", - "execution-checkpoint-v2.schema.json", - "core-step-result-v2.schema.json", + "migration-descriptor-v1.schema.json", + "provider-reference-v1.schema.json", + "public-host-request-v1.schema.json", + "public-host-response-v1.schema.json", + "recovery-operation-v1.schema.json", + "recovery-record-v1.schema.json", + "runtime-action-output-v1.schema.json", + "runtime-provider-descriptor-v1.schema.json", + "timer-helper-operation-v1.schema.json", ) -SPEC_COMMIT = "6796b554b976627c68fc746ec5629629aaf2b38e" +SPEC_COMMIT = "77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0" def _fetch(name: str) -> str: From 537c26e4df420a7487a27bad7120ad679ec20aff Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 15:00:10 +0000 Subject: [PATCH 03/32] Adapt native source-slot regression to recorded snapshots --- tests/test_runtime_providers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_runtime_providers.py b/tests/test_runtime_providers.py index 533a5c4..e3b6d9e 100644 --- a/tests/test_runtime_providers.py +++ b/tests/test_runtime_providers.py @@ -829,7 +829,7 @@ def test_source_declared_slots_do_not_hide_weak_provider(tmp_path: Path) -> None "class Provider:", "class Provider:\n" " __slots__ = ('guard_calls', 'action_calls', 'external_calls', " - "'irreversible_effects', 'external_effect_log')", + "'irreversible_effects', 'external_effect_log', 'guard_snapshot', 'action_snapshot')", ) ) bundle, registry = _modified_runtime_bundle(tmp_path, provider_source=source) From c4fdf0ee65d5db90c80dd44a05b8da69dbec4f1a Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 15:21:02 +0000 Subject: [PATCH 04/32] WIP retain compilation evidence and reject inert source locators --- README.md | 6 +- conformance/runtime_provider_adapter.py | 95 ++++++++++++--- src/determa/state/compilation.py | 146 ++++++++++++++---------- src/determa/state/definition.py | 3 + src/determa/state/runtime_providers.py | 73 +++++++----- tests/test_runtime_providers.py | 54 +++++++++ 6 files changed, 273 insertions(+), 104 deletions(-) diff --git a/README.md b/README.md index 5018d07..9c78281 100644 --- a/README.md +++ b/README.md @@ -329,7 +329,11 @@ a failed compare-and-swap; the engine does not retry it automatically. `compile_language_source(source, registry, manifest=manifest)` verifies the exact source and compiler closure, compiles disjoint regions in order, strictly loads the generated format-1 bundle, and checks the manifest fingerprint and proved source -capabilities. A generated CEL bundle restores without an installed compiler. +capabilities. Only executable grammar slots can be compiled; metadata and variable +values are inert. Every successful result retains sealed version-1 source and +manifest artifacts in `bundle.source_compilation`, including when no manifest was +supplied. That evidence discloses historical source guarantees independently of +the generated runtime profile. A generated CEL bundle restores without an installed compiler. Semantic inspection calls only a separately proved, bounded `inspect_guard` method; structural inspection never invokes a native provider. diff --git a/conformance/runtime_provider_adapter.py b/conformance/runtime_provider_adapter.py index 287df36..ff4c5f1 100644 --- a/conformance/runtime_provider_adapter.py +++ b/conformance/runtime_provider_adapter.py @@ -63,7 +63,7 @@ def _empty() -> dict[str, Any]: "result": "rejected", "code": None, "value": None, - "calls": {"guard": 0, "actions": 0, "inspect_guard": 0, "external": 0}, + "calls": {"guard": 0, "actions": 0, "inspect_guard": 0, "compile_region": 0, "external": 0}, "external_effects": [], "irreversible_side_effects": 0, "determa_state_committed": False, @@ -149,7 +149,9 @@ def _installed( module.compile_region, closure, capability_proof=lambda selected: ( - _PURE_CLAIMS if selected is module.compile_region else frozenset() + _PURE_CLAIMS + if selected is module.compile_region and not arguments.get("weak_compiler", False) + else frozenset() ), ) else: @@ -603,9 +605,27 @@ def execute(payload: dict[str, Any]) -> dict[str, Any]: source["artifact_digest"] = hash_value( [source["artifact_format"], "1", typed_value(source["content"])] ) - from determa.state.compilation import _artifact + if "invalid_slot" in arguments: + content = source["content"] + region = content["regions"][0] + if arguments["invalid_slot"] == "metadata_guard": + content["template"]["meta"] = {"guard": "true"} + region["locator"] = "/meta/guard" + else: + content["template"]["machines"][0]["root"]["variables"] = { + "data": {"type": "map", "init": {"action": []}} + } + region.update( + kind="actions", locator="/machines/0/root/variables/data/init/action" + ) + from determa.state.wire import hash_value, typed_value + + source["artifact_digest"] = hash_value( + [source["artifact_format"], "1", typed_value(content)] + ) + from determa.state.compilation import _source_preflight, _verify_compilation_manifest - _artifact(source, "language-source-v1.schema.json", "determa.language_source") + _source_preflight(source) observation["stages"].append("resolve_compiler_closure") registry, _module, loaded = _installed( root, payload, installed, arguments, compiler=True @@ -626,15 +646,30 @@ def execute(payload: dict[str, Any]) -> dict[str, Any]: ] else: manifest = None - bundle = compile_language_source( - source, - registry, - maximum_compilation_steps=arguments.get("maximum_compilation_steps", 1000), - ) - observation["stages"].append("strict_load") - if manifest is not None: - observation["stages"].append("verify_manifest") - bundle = compile_language_source(source, registry, manifest=manifest) + previous_profile = sys.getprofile() + + def trace(frame, event, arg): + if event == "call": + if frame.f_code is _module.compile_region.__code__: + observation["calls"]["compile_region"] += 1 + elif ( + frame.f_code is load_bundle.__code__ + and frame.f_back.f_code is compile_language_source.__code__ + ): + observation["stages"].append("strict_load") + elif frame.f_code is _verify_compilation_manifest.__code__: + observation["stages"].append("verify_manifest") + + sys.setprofile(trace) + try: + bundle = compile_language_source( + source, + registry, + manifest=manifest, + maximum_compilation_steps=arguments.get("maximum_compilation_steps", 1000), + ) + finally: + sys.setprofile(previous_profile) if "generated_bundle_file" in arguments: supplied = load_bundle( json.loads((root / arguments["generated_bundle_file"]).read_text()) @@ -647,9 +682,37 @@ def execute(payload: dict[str, Any]) -> dict[str, Any]: "on_events" ]["submit"]["guard"] } - observation["effective_capabilities"] = registry.compiler_capabilities( - source["content"]["regions"][0]["provider_reference"] - ) + assert bundle.source_compilation is not None + evidence = bundle.source_compilation["manifest"]["content"] + observation["effective_capabilities"] = evidence["source_capabilities"] + if arguments.get("without_manifest") and arguments.get("weak_compiler"): + generated_registry = RuntimeProviderRegistry() + generated = load_bundle(bundle.raw, runtime_providers=generated_registry) + created = create(generated, "order", "compiled-restore-root", "compiled-create", {}) + before_restore_calls = observation["calls"]["compile_region"] + sys.setprofile(trace) + try: + restored = restore_aggregate( + created["state"], + MemoryArtifactResolver(definitions={generated.fingerprint: generated}), + ) + finally: + sys.setprofile(previous_profile) + observation["value"].update( + source_artifact_digest=evidence["source_artifact_digest"], + compiler_providers=evidence["compiler_providers"], + generated_validated_bundle_fingerprint=evidence[ + "generated_validated_bundle_fingerprint" + ], + generated_runtime_capabilities=generated_registry.effective_capabilities( + generated.raw + ), + restored_runtime_capabilities=generated_registry.effective_capabilities( + restored.bundle.raw + ), + restore_compiler_calls=observation["calls"]["compile_region"] + - before_restore_calls, + ) else: observation["stages"].append( "resolve_runtime_closure" if operation == "restore" else "resolve_closure" diff --git a/src/determa/state/compilation.py b/src/determa/state/compilation.py index ce711f6..b79324e 100644 --- a/src/determa/state/compilation.py +++ b/src/determa/state/compilation.py @@ -5,11 +5,17 @@ import copy import json from collections.abc import Mapping +from dataclasses import replace from pathlib import Path from typing import Any from .definition import Bundle, load_bundle -from .runtime_providers import RuntimeProviderError, RuntimeProviderRegistry, _reference_key +from .runtime_providers import ( + RuntimeProviderError, + RuntimeProviderRegistry, + _executable_slots, + _reference_key, +) from .wire import _schema_registry, hash_value, typed_value @@ -64,14 +70,7 @@ def _slot(template: dict[str, Any], tokens: tuple[str, ...]) -> tuple[Any, str]: return parent, key -def compile_language_source( - source: Mapping[str, Any], - registry: RuntimeProviderRegistry, - *, - manifest: Mapping[str, Any] | None = None, - maximum_compilation_steps: int = 1000, -) -> Bundle: - """Verify provenance, compile disjoint regions, and strictly load the generated bundle.""" +def _source_preflight(source: Mapping[str, Any]) -> tuple[dict[str, Any], list[tuple[str, ...]]]: source_doc = _artifact( dict(source), "language-source-v1.schema.json", "determa.language_source" ) @@ -82,6 +81,48 @@ def compile_language_source( a != b and (a[: len(b)] == b or b[: len(a)] == a) for a in locations for b in locations ): raise RuntimeProviderError("language_compilation_failed") + generated = content["template"] + try: + slots = {location: (kind, value) for kind, location, value in _executable_slots(generated)} + except (AttributeError, TypeError, ValueError) as exc: + raise RuntimeProviderError("language_compilation_failed") from exc + for location, region in zip(locations, regions, strict=True): + slot = slots.get(location) + if ( + slot is None + or slot[0] != region["kind"] + or ( + type(slot[1]) is not str if region["kind"] == "guard" else type(slot[1]) is not list + ) + ): + raise RuntimeProviderError("language_compilation_failed") + return source_doc, locations + + +def _verify_compilation_manifest(manifest: Mapping[str, Any], computed: dict[str, Any]) -> None: + provided = _artifact( + dict(manifest), "compilation-manifest-v1.schema.json", "determa.compilation_manifest" + ) + if provided != computed: + raise RuntimeProviderError("language_compilation_failed") + + +def compile_language_source( + source: Mapping[str, Any], + registry: RuntimeProviderRegistry, + *, + manifest: Mapping[str, Any] | None = None, + maximum_compilation_steps: int = 1000, +) -> Bundle: + """Compile grammar slots and retain sealed version-1 source/manifest evidence. + + ``Bundle.source_compilation`` describes historical source guarantees; the + generated bundle keeps its independently verified runtime capability profile. + """ + source_doc, locations = _source_preflight(source) + content = source_doc["content"] + regions = content["regions"] + generated = copy.deepcopy(content["template"]) dependencies = content["dependencies"] if dependencies != sorted(dependencies, key=_reference_key) or len( {_reference_key(item) for item in dependencies} @@ -96,17 +137,9 @@ def compile_language_source( or not closure.manifest_verified() ): raise RuntimeProviderError("runtime_provider_unavailable") - generated = copy.deepcopy(content["template"]) - for location, region in zip(locations, regions, strict=True): - parent, key = _slot(generated, location) - value = parent[_array_index(key)] if isinstance(parent, list) else parent[key] - if (region["kind"] == "guard" and (key != "guard" or type(value) is not str)) or ( - region["kind"] == "actions" - and (key not in {"action", "entry", "exit"} or type(value) is not list) - ): - raise RuntimeProviderError("language_compilation_failed") compilers = [region["provider_reference"] for region in regions] compiled = [registry.compiler(region["provider_reference"]) for region in regions] + compiler_claims_before = [registry.compiler_capabilities(reference) for reference in compilers] for location, region, compiler in zip(locations, regions, compiled, strict=True): if maximum_compilation_steps <= 0: raise RuntimeProviderError("language_compilation_limit_exceeded") @@ -124,48 +157,43 @@ def compile_language_source( bundle = load_bundle(generated, runtime_providers=registry) except Exception as exc: raise RuntimeProviderError("language_compilation_failed") from exc - if manifest is not None: - manifest_doc = _artifact( - dict(manifest), "compilation-manifest-v1.schema.json", "determa.compilation_manifest" - ) - record = manifest_doc["content"] - expected_closure = sorted( - { - *(_reference_key(item) for item in dependencies), - *(_reference_key(item) for item in compilers), - } - ) - effective = dict.fromkeys( - ( - "deterministic", - "pure", - "portable", - "semantically_introspectable", - "process_contained", - ), - True, - ) - effective["external_io_capable"] = False - for reference in compilers: - claims = registry.compiler_capabilities(reference) - for name in effective: - effective[name] = ( - effective[name] or claims[name] - if name == "external_io_capable" - else effective[name] and claims[name] - ) - runtime_claims = registry.effective_capabilities(bundle.raw) + expected_closure = sorted( + { + *(_reference_key(item) for item in dependencies), + *(_reference_key(item) for item in compilers), + } + ) + effective = dict.fromkeys( + ("deterministic", "pure", "portable", "semantically_introspectable", "process_contained"), + True, + ) + effective["external_io_capable"] = False + for claims in [ + *compiler_claims_before, + *(registry.compiler_capabilities(reference) for reference in compilers), + registry.effective_capabilities(bundle.raw), + ]: for name in effective: effective[name] = ( - effective[name] or runtime_claims[name] + effective[name] or claims[name] if name == "external_io_capable" - else effective[name] and runtime_claims[name] + else effective[name] and claims[name] ) - if ( - record["source_artifact_digest"] != source_doc["artifact_digest"] - or [_reference_key(item) for item in record["compiler_providers"]] != expected_closure - or record["generated_validated_bundle_fingerprint"] != bundle.fingerprint - or record["source_capabilities"] != effective - ): - raise RuntimeProviderError("language_compilation_failed") - return bundle + record = { + "source_artifact_digest": source_doc["artifact_digest"], + "compiler_providers": [ + dict(zip(("identifier", "version", "content_digest"), item, strict=True)) + for item in expected_closure + ], + "generated_validated_bundle_fingerprint": bundle.fingerprint, + "source_capabilities": effective, + } + manifest_doc = { + "artifact_format": "determa.compilation_manifest", + "artifact_schema_version": 1, + "content": record, + "artifact_digest": hash_value(["determa.compilation_manifest", "1", typed_value(record)]), + } + if manifest is not None: + _verify_compilation_manifest(manifest, manifest_doc) + return replace(bundle, source_compilation={"source": source_doc, "manifest": manifest_doc}) diff --git a/src/determa/state/definition.py b/src/determa/state/definition.py index 5b63565..a40a91e 100644 --- a/src/determa/state/definition.py +++ b/src/determa/state/definition.py @@ -163,6 +163,9 @@ class Bundle: required_capabilities: frozenset[str] = dataclass_field( default=frozenset(), compare=False, repr=False ) + source_compilation: dict[str, Any] | None = dataclass_field( + default=None, compare=False, repr=False + ) def verify_runtime_policy(self) -> None: """Recheck exact provider closure, current health, and host requirements.""" diff --git a/src/determa/state/runtime_providers.py b/src/determa/state/runtime_providers.py index 9adec2b..b38e6ab 100644 --- a/src/determa/state/runtime_providers.py +++ b/src/determa/state/runtime_providers.py @@ -27,35 +27,52 @@ def _runtime_bindings( ) -> Iterator[tuple[str, Mapping[str, Any]]]: """Visit only executable grammar slots; metadata and values remain inert.""" - def actions(items: Any) -> Iterator[tuple[str, Mapping[str, Any]]]: - for action in items or []: - if "provider_actions" in action: - yield "actions", action["provider_actions"] - - def transitions(value: Any) -> Iterator[tuple[str, Mapping[str, Any]]]: - for transition in value if isinstance(value, list) else [value]: - if not transition: + for kind, _, value in _executable_slots(definition): + if kind == "guard" and isinstance(value, Mapping) and "provider" in value: + yield kind, value["provider"] + elif kind == "actions" and isinstance(value, list): + for action in value: + if isinstance(action, Mapping) and "provider_actions" in action: + yield kind, action["provider_actions"] + + +def _executable_slots( + definition: Mapping[str, Any], +) -> Iterator[tuple[str, tuple[str, ...], Any]]: + """Enumerate grammar-owned guards and action blocks, never arbitrary values.""" + + def transitions( + value: Any, path: tuple[str, ...] + ) -> Iterator[tuple[str, tuple[str, ...], Any]]: + entries = enumerate(value) if isinstance(value, list) else [(None, value)] + for index, transition in entries: + if not isinstance(transition, Mapping): continue - guard = transition.get("guard") - if isinstance(guard, Mapping) and "provider" in guard: - yield "guard", guard["provider"] - yield from actions(transition.get("action")) - - def state(value: Mapping[str, Any]) -> Iterator[tuple[str, Mapping[str, Any]]]: - yield from actions(value.get("entry")) - yield from actions(value.get("exit")) - yield from transitions(value.get("initial")) - yield from transitions(value.get("choice")) - for handler in (value.get("on_events") or {}).values(): - yield from transitions(handler) - for child in (value.get("states") or {}).values(): - yield from state(child) - for component in value.get("components") or []: - if "root" in component: - yield from state(component["root"]) - - for machine in definition.get("machines") or []: - yield from state(machine["root"]) + current = path if index is None else (*path, str(index)) + if "guard" in transition: + yield "guard", (*current, "guard"), transition["guard"] + if "action" in transition: + yield "actions", (*current, "action"), transition["action"] + + def state(value: Any, path: tuple[str, ...]) -> Iterator[tuple[str, tuple[str, ...], Any]]: + if not isinstance(value, Mapping): + return + for name in ("entry", "exit"): + if name in value: + yield "actions", (*path, name), value[name] + for name in ("initial", "choice"): + yield from transitions(value.get(name), (*path, name)) + for name, handler in (value.get("on_events") or {}).items(): + yield from transitions(handler, (*path, "on_events", name)) + for name, child in (value.get("states") or {}).items(): + yield from state(child, (*path, "states", name)) + for index, component in enumerate(value.get("components") or []): + if isinstance(component, Mapping) and "root" in component: + yield from state(component["root"], (*path, "components", str(index), "root")) + + for index, machine in enumerate(definition.get("machines") or []): + if isinstance(machine, Mapping): + yield from state(machine.get("root"), ("machines", str(index), "root")) SourceIdentityVerifier = Callable[[Path, Any], bool] diff --git a/tests/test_runtime_providers.py b/tests/test_runtime_providers.py index e3b6d9e..687520b 100644 --- a/tests/test_runtime_providers.py +++ b/tests/test_runtime_providers.py @@ -196,11 +196,65 @@ def test_exact_compilation_and_manifest_fingerprint() -> None: manifest = json.loads((_PROFILE / "source-manifest.json").read_text()) bundle = compile_language_source(source, registry, manifest=manifest) assert bundle.fingerprint == manifest["content"]["generated_validated_bundle_fingerprint"] + assert bundle.source_compilation == {"source": source, "manifest": manifest} manifest["content"]["generated_validated_bundle_fingerprint"] = "sha256:" + "0" * 64 with pytest.raises(RuntimeProviderError, match="language_compilation_failed"): compile_language_source(source, registry, manifest=manifest) +@pytest.mark.parametrize("slot", ["metadata", "value"]) +def test_compiler_rejects_inert_slot_before_resolution(slot: str) -> None: + _, source = _installed_compiler(_PROFILE) + region = source["content"]["regions"][0] + template = source["content"]["template"] + if slot == "metadata": + template["meta"] = {"guard": "true"} + region["locator"] = "/meta/guard" + else: + template["machines"][0]["root"]["variables"] = { + "data": {"type": "map", "init": {"action": []}} + } + region.update(kind="actions", locator="/machines/0/root/variables/data/init/action") + source["artifact_digest"] = hash_value( + [source["artifact_format"], "1", typed_value(source["content"])] + ) + # No dependency/compiler is installed: grammar rejection must precede resolution. + with pytest.raises(RuntimeProviderError, match="language_compilation_failed"): + compile_language_source(source, RuntimeProviderRegistry()) + + +def test_weak_compiler_retains_source_evidence_without_manifest() -> None: + registry, source = _installed_compiler(_PROFILE) + key = tuple( + source["content"]["regions"][0]["provider_reference"][name] + for name in ("identifier", "version", "content_digest") + ) + registry._compiler_proofs[key] = None + bundle = compile_language_source(source, registry) + assert bundle.source_compilation is not None + evidence = bundle.source_compilation + assert evidence["source"] == source + assert evidence["manifest"]["content"]["source_capabilities"] == { + "deterministic": False, + "pure": False, + "portable": False, + "semantically_introspectable": False, + "process_contained": False, + "external_io_capable": True, + } + assert ( + evidence["manifest"]["content"]["generated_validated_bundle_fingerprint"] + == bundle.fingerprint + ) + generated = load_bundle(bundle.raw) + created = create(generated, "order", "generated-root", "generated-create", {}) + restored = restore_aggregate( + created["state"], MemoryArtifactResolver(definitions={generated.fingerprint: generated}) + ) + assert restored.bundle.source_compilation is None + assert restored.bundle.fingerprint == bundle.fingerprint + + def test_compiler_source_drift_refused_before_callback(tmp_path: Path) -> None: shutil.copytree(_PROFILE, tmp_path / "fixture") root = tmp_path / "fixture" From 49686995366e1daed2db3598089935877ea5c32a Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 15:34:51 +0000 Subject: [PATCH 05/32] WIP support typed native component environment forwarding --- conformance/runtime_provider_adapter.py | 72 +++++++++++++++++++++++++ src/determa/state/engine.py | 30 ++++++++++- 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/conformance/runtime_provider_adapter.py b/conformance/runtime_provider_adapter.py index ff4c5f1..affdfc8 100644 --- a/conformance/runtime_provider_adapter.py +++ b/conformance/runtime_provider_adapter.py @@ -184,6 +184,11 @@ def _installed( "external_io": bool(arguments.get("action_external_io", False)), **({"repeat_send": True} if arguments.get("repeat_send") else {}), **({"mixed_send": True} if arguments.get("mixed_send") else {}), + **( + {"environment_send": arguments["environment_send"]} + if "environment_send" in arguments + else {} + ), } ), ) @@ -221,6 +226,24 @@ def _state(aggregate: dict[str, Any]) -> dict[str, Any]: } +def _component_states(aggregate: dict[str, Any]) -> dict[str, Any]: + result = {} + for runtime in aggregate["runtimes"]: + component = runtime["target_identity"].get("component") + if component is not None: + observed = _state({**aggregate, "root_runtime_id": runtime["runtime_id"]}) + result[component["component_id"]] = { + key: observed[key] + for key in ( + "status", + "variables", + "ready_mailbox_length", + "deferred_mailbox_length", + ) + } + return result + + def _run_step( bundle: Any, registry: RuntimeProviderRegistry, @@ -271,6 +294,12 @@ def _run_step( observation["result"] = "faulted" observation["code"] = result["fault"]["code"] observation["state_after"] = observation["state_before"] + if "environment_send" in request["arguments"]: + observation["value"] = { + "source_locator": result["fault"]["source_locator"], + "component_states_before": _component_states(aggregate), + "component_states_after": _component_states(result["state"]), + } if request["arguments"].get("invalid_output") or request["arguments"].get( "destroyed_write" ): @@ -285,6 +314,49 @@ def _run_step( observation["determa_state_committed"] = True observation["state_after"] = _state(result["state"]) observation["value"] = {"emissions": len(result["emissions"])} + if "environment_send" in request["arguments"]: + before_child = next( + item + for item in aggregate["runtimes"] + if item.get("target_identity", {}).get("component", {}).get("component_id") == "replica" + ) + after_child = next( + item + for item in result["state"]["runtimes"] + if item.get("target_identity", {}).get("component", {}).get("component_id") == "replica" + ) + internal = next( + item for item in result["emissions"] if item.get("kind") == "internal_mailbox" + ) + emission = next( + item["envelope"] + for item in after_child["ready_mailbox"] + if item["envelope"]["event_id"] == internal["event_id"] + ) + observation["value"].update( + forwarded_event={ + "event": emission["event"], + "component_id": after_child["target_identity"]["component"]["component_id"], + "payload": emission["payload"], + }, + component_variables_before=_state( + {**aggregate, "root_runtime_id": before_child["runtime_id"]} + )["variables"], + component_ready_before_delivery=len(after_child["ready_mailbox"]), + ) + delivered = step(result["state"], after_child["runtime_id"], resolver) + observation["stages"].append("deliver_env") + refreshed = next( + item + for item in delivered["state"]["runtimes"] + if item["runtime_id"] == after_child["runtime_id"] + ) + observation["value"].update( + component_variables_after=_state( + {**delivered["state"], "root_runtime_id": refreshed["runtime_id"]} + )["variables"], + component_ready_after_delivery=len(refreshed["ready_mailbox"]), + ) if "accepted" in observation["state_after"]["variables"]: observation["value"]["accepted"] = observation["state_after"]["variables"]["accepted"] if observation["state_after"]["status"] == "completed": diff --git a/src/determa/state/engine.py b/src/determa/state/engine.py index cb561f7..0e2faa5 100644 --- a/src/determa/state/engine.py +++ b/src/determa/state/engine.py @@ -2200,12 +2200,22 @@ def send( if literal_values: from .wire import decoded_typed_value - if send["event"] == "env" or declaration is None: + if declaration is None and send["event"] != "env": raise StepFault(FaultCode.ACTION_FAULT, pointer) decoded = decoded_typed_value(send["payload"]) if not isinstance(decoded, dict): raise StepFault(FaultCode.ACTION_FAULT, pointer) payload_values = decoded + if send["event"] == "env" and ( + "targets" in send + or "correlation_id" in send + or not isinstance(send.get("to"), dict) + or set(send["to"]) != {"component"} + or set(payload_values) != {"changed"} + or not isinstance(payload_values["changed"], dict) + or not payload_values["changed"] + ): + raise StepFault(FaultCode.ACTION_FAULT, pointer) elif send["event"] == "env": changed_expression = payload_expressions["changed"] payload_values["changed"] = self.evaluate( @@ -2263,6 +2273,24 @@ def send( self.resolve_send_target(runtime, target_spec, value, pointer, index, "targets" in send) for index, (target_spec, value) in enumerate(evaluated_targets) ] + if literal_values and send["event"] == "env": + component_target = resolved[0] + if not isinstance(component_target, dict): + raise StepFault(FaultCode.ACTION_FAULT, pointer) + child = self.state["runtimes"][_target_runtime_id(component_target)] + root = _pointer_get(self.bundle.raw, child["root_pointer"]) + external = { + name: item + for name, item in (root.get("variables") or {}).items() + if item.get("external") is True + } + changed = normalized_payload["changed"] + if set(changed) - set(external): + raise StepFault(FaultCode.ACTION_FAULT, pointer) + normalized_payload["changed"] = { + name: _normalize_value(value, str(external[name]["type"])) + for name, value in changed.items() + } if ( literal_values and declaration is not None From befa708f75420ef2475c5dbdaefc3f2b3394eefc Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 15:38:57 +0000 Subject: [PATCH 06/32] Reverify each compiler region and pin reviewed provider coverage --- AGENTS.md | 4 +-- README.md | 4 +-- conformance/pins.py | 2 +- src/determa/state/compilation.py | 6 +++-- tests/test_runtime_providers.py | 44 ++++++++++++++++++++++++++++++++ 5 files changed, 53 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 61b5cd9..eba3e97 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,10 +12,10 @@ The implementation is conformant only when it passes the language-neutral suite. The implementation uses these immutable inputs: - specification: `77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0`; -- conformance: `cf7e673b26ebb162dfdc54d9d235300cd3350438` (99 format-1 +- conformance: `c6949440f5b9fd6e06ea5571f4e01681dd59ee04` (99 format-1 core cases, 162 version-1 vectors, 142 durable-host vectors, 49 core inspection vectors, 47 extension negotiation vectors, 7 optional native-provider vectors, - 41 runtime-provider vectors, and 487 JSON artifacts). + 52 runtime-provider vectors, and 487 JSON artifacts). The package metadata is `0.3.0`. diff --git a/README.md b/README.md index 9c78281..596bd4e 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ This implementation supports Determa State `format: 1` at specification commit `77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0`. Correctness is determined by 99 format-1 core cases, 162 version-1 vectors, 142 durable-host vectors, 49 core inspection vectors, 47 extension negotiation vectors, 7 native guard-provider -inspection vectors, 41 runtime-provider vectors, and 487 manifest artifacts at -conformance commit `cf7e673b26ebb162dfdc54d9d235300cd3350438`. The native +inspection vectors, 52 runtime-provider vectors, and 487 manifest artifacts at +conformance commit `c6949440f5b9fd6e06ea5571f4e01681dd59ee04`. The native runtime-provider profile is optional and is exercised through its production adapter. The package metadata is `0.3.0`. Artifact and checkpoint schema version 1 is the diff --git a/conformance/pins.py b/conformance/pins.py index e5e259c..0f9c49f 100644 --- a/conformance/pins.py +++ b/conformance/pins.py @@ -4,7 +4,7 @@ from pathlib import Path -CONFORMANCE_COMMIT = "cf7e673b26ebb162dfdc54d9d235300cd3350438" +CONFORMANCE_COMMIT = "c6949440f5b9fd6e06ea5571f4e01681dd59ee04" SPEC_COMMIT = "77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0" ROOT = Path(__file__).resolve().parent.parent diff --git a/src/determa/state/compilation.py b/src/determa/state/compilation.py index b79324e..8b5d6af 100644 --- a/src/determa/state/compilation.py +++ b/src/determa/state/compilation.py @@ -138,12 +138,14 @@ def compile_language_source( ): raise RuntimeProviderError("runtime_provider_unavailable") compilers = [region["provider_reference"] for region in regions] - compiled = [registry.compiler(region["provider_reference"]) for region in regions] + for region in regions: + registry.compiler(region["provider_reference"]) compiler_claims_before = [registry.compiler_capabilities(reference) for reference in compilers] - for location, region, compiler in zip(locations, regions, compiled, strict=True): + for location, region in zip(locations, regions, strict=True): if maximum_compilation_steps <= 0: raise RuntimeProviderError("language_compilation_limit_exceeded") maximum_compilation_steps -= 1 + compiler = registry.compiler(region["provider_reference"]) try: replacement = compiler(region["source"]) except Exception as exc: diff --git a/tests/test_runtime_providers.py b/tests/test_runtime_providers.py index 687520b..570a44e 100644 --- a/tests/test_runtime_providers.py +++ b/tests/test_runtime_providers.py @@ -265,6 +265,50 @@ def test_compiler_source_drift_refused_before_callback(tmp_path: Path) -> None: compile_language_source(source, registry) +def test_compiler_rebinding_between_regions_refused_before_substitute_runs(tmp_path: Path) -> None: + body = (_PROFILE / "provider/test_provider.py").read_text().split("def compile_region(")[0] + body += """ +calls = [] +def substitute(source): + calls.append('substitute') + return 'event.payload.approved' +def compile_region(source): + calls.append('original') + compile_region.__code__ = substitute.__code__ + return 'event.payload.approved' +""" + bundle, registry = _modified_runtime_bundle(tmp_path, provider_source=body) + active = registry.resolve("guard", _fixture_guard_binding(bundle)).provider + module = sys.modules[type(active).__module__] + closure = SourceClosure( + tmp_path / "fixture", + ("provider/test_provider.py", "provider/test_provider.rs"), + "provider-closure.json", + _DOMAIN, + "provider/test_provider.py", + ) + reference = { + "identifier": "test-mutating-compiler", + "version": "1.0.0", + "content_digest": closure.digest(), + } + registry.register_compiler(reference, module.compile_region, closure) + source = json.loads((_PROFILE / "source-package.json").read_text()) + content = source["content"] + content["dependencies"] = [] + second = json.loads(json.dumps(content["template"]["machines"][0])) + second["machine_id"] = "second" + content["template"]["machines"].append(second) + content["regions"][0]["provider_reference"] = reference + region = json.loads(json.dumps(content["regions"][0])) + region["locator"] = region["locator"].replace("/machines/0/", "/machines/1/") + content["regions"].append(region) + source["artifact_digest"] = hash_value([source["artifact_format"], "1", typed_value(content)]) + with pytest.raises(RuntimeProviderError, match="runtime_provider_unavailable"): + compile_language_source(source, registry) + assert module.calls == ["original"] + + def test_compiler_manifest_drift_refused_at_use(tmp_path: Path) -> None: shutil.copytree(_PROFILE, tmp_path / "fixture") root = tmp_path / "fixture" From d1d626a15a909d176f915013d88c0fa7a6ba7597 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 15:44:52 +0000 Subject: [PATCH 07/32] Align CI pins and exercise sealed manifest fingerprint mismatch --- .github/workflows/test.yml | 12 ++++++------ conformance/runtime_provider_adapter.py | 5 +++++ tests/test_runtime_providers.py | 3 +++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b2fbdbc..28d7888 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,13 +53,13 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: affe3fe3bcc4d13fa7c5374471568e94af36f0d1 + ref: c6949440f5b9fd6e06ea5571f4e01681dd59ee04 path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-spec - ref: 86bb88dd21cb1f799eefe5020b6e49dabf6e7225 + ref: 77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0 path: .pinned/determa-state-spec - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: @@ -88,13 +88,13 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: affe3fe3bcc4d13fa7c5374471568e94af36f0d1 + ref: c6949440f5b9fd6e06ea5571f4e01681dd59ee04 path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-spec - ref: 86bb88dd21cb1f799eefe5020b6e49dabf6e7225 + ref: 77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0 path: .pinned/determa-state-spec - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: @@ -126,13 +126,13 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: affe3fe3bcc4d13fa7c5374471568e94af36f0d1 + ref: c6949440f5b9fd6e06ea5571f4e01681dd59ee04 path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-spec - ref: 86bb88dd21cb1f799eefe5020b6e49dabf6e7225 + ref: 77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0 path: .pinned/determa-state-spec - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: diff --git a/conformance/runtime_provider_adapter.py b/conformance/runtime_provider_adapter.py index affdfc8..ad32eaf 100644 --- a/conformance/runtime_provider_adapter.py +++ b/conformance/runtime_provider_adapter.py @@ -716,6 +716,11 @@ def execute(payload: dict[str, Any]) -> dict[str, Any]: manifest["content"]["generated_validated_bundle_fingerprint"] = arguments[ "manifest_fingerprint_override" ] + from determa.state.wire import hash_value, typed_value + + manifest["artifact_digest"] = hash_value( + [manifest["artifact_format"], "1", typed_value(manifest["content"])] + ) else: manifest = None previous_profile = sys.getprofile() diff --git a/tests/test_runtime_providers.py b/tests/test_runtime_providers.py index 570a44e..5f9b576 100644 --- a/tests/test_runtime_providers.py +++ b/tests/test_runtime_providers.py @@ -198,6 +198,9 @@ def test_exact_compilation_and_manifest_fingerprint() -> None: assert bundle.fingerprint == manifest["content"]["generated_validated_bundle_fingerprint"] assert bundle.source_compilation == {"source": source, "manifest": manifest} manifest["content"]["generated_validated_bundle_fingerprint"] = "sha256:" + "0" * 64 + manifest["artifact_digest"] = hash_value( + [manifest["artifact_format"], "1", typed_value(manifest["content"])] + ) with pytest.raises(RuntimeProviderError, match="language_compilation_failed"): compile_language_source(source, registry, manifest=manifest) From 42d589aa9c487f136bcf1e4e23d2abf867b4ba16 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 16:19:37 +0000 Subject: [PATCH 08/32] Add unfinished durable foreground timer helper foundation --- scripts/sync_schema.py | 1 + .../state/data/timer-record-v1.schema.json | 177 +++++++++ src/determa/state/timers.py | 369 ++++++++++++++++++ tests/test_timers.py | 160 ++++++++ 4 files changed, 707 insertions(+) create mode 100644 src/determa/state/data/timer-record-v1.schema.json create mode 100644 src/determa/state/timers.py create mode 100644 tests/test_timers.py diff --git a/scripts/sync_schema.py b/scripts/sync_schema.py index 28250cb..c17377a 100644 --- a/scripts/sync_schema.py +++ b/scripts/sync_schema.py @@ -53,6 +53,7 @@ "runtime-action-output-v1.schema.json", "runtime-provider-descriptor-v1.schema.json", "timer-helper-operation-v1.schema.json", + "timer-record-v1.schema.json", ) SPEC_COMMIT = "77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0" diff --git a/src/determa/state/data/timer-record-v1.schema.json b/src/determa/state/data/timer-record-v1.schema.json new file mode 100644 index 0000000..06a7f57 --- /dev/null +++ b/src/determa/state/data/timer-record-v1.schema.json @@ -0,0 +1,177 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://determa.dev/state/schema/timer-record-v1.schema.json", + "title": "Separate portable timer records artifact version 1", + "type": "object", + "properties": { + "timer_artifact_format": { + "const": "determa.timer_records" + }, + "timer_artifact_schema_version": { + "const": 1 + }, + "records": { + "type": "array", + "items": { + "type": "object", + "properties": { + "scope_identity": { + "type": "string", + "minLength": 1 + }, + "root_instance_id": { + "type": "string", + "minLength": 1 + }, + "root_runtime_id": { + "type": "string", + "minLength": 1 + }, + "timer_id": { + "type": "string", + "minLength": 1 + }, + "schedule_request_digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "clock_basis": { + "const": "unix_nanoseconds" + }, + "deadline_at": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*|-[1-9][0-9]*)$" + }, + "event_name": { + "type": "string", + "minLength": 1 + }, + "payload": { + "$ref": "aggregate-state-v1.schema.json#/$defs/typedMap" + }, + "correlation_id": { + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "null" + } + ] + }, + "event_id": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "state": { + "enum": [ + "pending", + "claimed", + "fired", + "cancelled" + ] + }, + "revision": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "attempt_fence": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "worker_principal": { + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "null" + } + ] + }, + "expires_at": { + "oneOf": [ + { + "type": "string", + "pattern": "^(0|[1-9][0-9]*|-[1-9][0-9]*)$" + }, + { + "type": "null" + } + ] + }, + "admission_receipt_digest": { + "oneOf": [ + { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "scope_identity", + "root_instance_id", + "root_runtime_id", + "timer_id", + "schedule_request_digest", + "clock_basis", + "deadline_at", + "event_name", + "payload", + "correlation_id", + "event_id", + "state", + "revision", + "attempt_fence", + "worker_principal", + "expires_at", + "admission_receipt_digest" + ], + "additionalProperties": false + } + }, + "operation_receipts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "operation_id": { + "type": "string", + "minLength": 1 + }, + "request_digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "result": { + "$ref": "timer-helper-operation-v1.schema.json#/$defs/result" + } + }, + "required": [ + "operation_id", + "request_digest", + "result" + ], + "additionalProperties": false + } + }, + "timer_artifact_digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + } + }, + "required": [ + "timer_artifact_format", + "timer_artifact_schema_version", + "records", + "operation_receipts", + "timer_artifact_digest" + ], + "additionalProperties": false +} diff --git a/src/determa/state/timers.py b/src/determa/state/timers.py new file mode 100644 index 0000000..1a0cc88 --- /dev/null +++ b/src/determa/state/timers.py @@ -0,0 +1,369 @@ +"""Explicit foreground SQLite timer records, separate from portable checkpoints. + +This implementation checkpoint supports durable schedule/cancel/read operations. +Fire, coordinated admission, verified provider installation and archive integration +remain unfinished; no completed timer profile is advertised yet. +""" + +from __future__ import annotations + +import copy +import json +import re +import sqlite3 +from collections.abc import Callable, Mapping +from contextlib import closing +from pathlib import Path +from typing import Any + +from .definition import Bundle +from .engine import _normalize_payload +from .wire import _schema_registry, canonical_bytes, decoded_typed_value, hash_value + +_TIME = re.compile(r"(?:0|-?[1-9][0-9]*)\Z") +_TABLE = ( + "CREATE TABLE determa_timer_helpers " + "(scope_identity TEXT PRIMARY KEY NOT NULL, document BLOB NOT NULL)" +) +_IDENTITIES = ( + "operation", + "operation_id", + "scope_identity", + "root_instance_id", + "root_runtime_id", + "timer_id", +) + + +class TimerError(ValueError): + def __init__(self, code: str) -> None: + self.code = code + super().__init__(code) + + +def timer_request_digest(request: Mapping[str, Any]) -> str: + return hash_value( + [ + "determa-timer-request-1", + {key: value for key, value in request.items() if key != "request_digest"}, + ] + ) + + +def seal_timer_records(artifact: Mapping[str, Any]) -> dict[str, Any]: + body = copy.deepcopy( + {key: value for key, value in artifact.items() if key != "timer_artifact_digest"} + ) + return {**body, "timer_artifact_digest": hash_value(["determa-timer-artifact-1", body])} + + +def _validator(filename: str, definition: str | None = None) -> Any: + from jsonschema import Draft202012Validator + from referencing import Resource + + schema = json.loads((Path(__file__).parent / "data" / filename).read_text()) + registry = _schema_registry() + for name in ("timer-helper-operation-v1.schema.json", "timer-record-v1.schema.json"): + resource = json.loads((Path(__file__).parent / "data" / name).read_text()) + registry = registry.with_resource(resource["$id"], Resource.from_contents(resource)) + if definition is not None: + schema = {"$ref": schema["$id"] + "#/$defs/" + definition} + return Draft202012Validator(schema, registry=registry) + + +def _time(value: Any, *, duration: bool = False) -> int: + if type(value) is not str or not _TIME.fullmatch(value): + raise TimerError("invalid_timer_time") + parsed = int(value) + if not (0 if duration else -(2**63)) <= parsed <= 2**63 - 1: + raise TimerError("invalid_timer_time") + return parsed + + +def _result( + request: Mapping[str, Any] | None, record: dict[str, Any] | None, code: str | None = None +) -> dict[str, Any]: + result: dict[str, Any] = { + "interface": "determa.timer_helper", + "interface_version": 1, + **{key: request[key] if request is not None else None for key in _IDENTITIES}, + "status": "accepted" if code is None else "rejected", + "record_revision": record["revision"] if record else None, + "attempt_fence": record["attempt_fence"] + if record and record["attempt_fence"] != "0" + else None, + "clock_basis": "unix_nanoseconds", + "expires_at": None, + "event_id": record["event_id"] if record else None, + "delivery_state": "admitted" if record and record["admission_receipt_digest"] else "none", + "error_code": code, + "result_digest": None, + } + if code is None and request is not None: + result["result_digest"] = hash_value( + [ + "determa-timer-result-1", + request["request_digest"], + {key: value for key, value in result.items() if key != "result_digest"}, + ] + ) + return result + + +class SQLiteTimerHelper: + """One configured scope and immutable target incarnation; no background work. + + Principals and the clock are trusted host configuration, never request fields. + Schema setup is explicit. Records and exact successful receipts share one native + SQLite commit; a refusal does not write a receipt or change the artifact. + """ + + def __init__( + self, + path: str | Path, + bundle: Bundle, + *, + scope_identity: str, + root_instance_id: str, + root_runtime_id: str, + principals: frozenset[str], + trusted_clock: Callable[[], str], + ) -> None: + if ( + str(path) == ":memory:" + or not str(path) + or not all((scope_identity, root_instance_id, root_runtime_id)) + or not principals + ): + raise ValueError( + "persistent storage and a configured scope, target and principals are required" + ) + self.path = str(Path(path).resolve()) + self.bundle = bundle + self.scope_identity = scope_identity + self.root_instance_id = root_instance_id + self.root_runtime_id = root_runtime_id + self.principals = frozenset(principals) + self.trusted_clock = trusted_clock + + def _connect(self) -> sqlite3.Connection: + connection = sqlite3.connect(self.path, isolation_level=None, timeout=30) + connection.execute("PRAGMA journal_mode=WAL") + connection.execute("PRAGMA synchronous=FULL") + return connection + + def setup_schema(self) -> None: + with closing(self._connect()) as connection: + connection.execute(_TABLE.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1)) + self.validate_schema() + + def _schema(self, connection: sqlite3.Connection) -> None: + rows = connection.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='determa_timer_helpers'" + ).fetchall() + if rows != [(_TABLE,)] or connection.execute("PRAGMA integrity_check").fetchone() != ( + "ok", + ): + raise TimerError("timer_capability_mismatch") + + def validate_schema(self) -> None: + with closing(self._connect()) as connection: + self._schema(connection) + + def _load(self, connection: sqlite3.Connection) -> dict[str, Any]: + row = connection.execute( + "SELECT document FROM determa_timer_helpers WHERE scope_identity=?", + (self.scope_identity,), + ).fetchone() + if row is None: + return seal_timer_records( + { + "timer_artifact_format": "determa.timer_records", + "timer_artifact_schema_version": 1, + "records": [], + "operation_receipts": [], + } + ) + artifact: dict[str, Any] = json.loads(bytes(row[0])) + if ( + not _validator("timer-record-v1.schema.json").is_valid(artifact) + or seal_timer_records(artifact) != artifact + ): + raise TimerError("timer_capability_mismatch") + if any(record["scope_identity"] != self.scope_identity for record in artifact["records"]): + raise TimerError("timer_capability_mismatch") + return artifact + + def snapshot(self) -> dict[str, Any]: + with closing(self._connect()) as connection: + self._schema(connection) + return self._load(connection) + + def _persist(self, connection: sqlite3.Connection, artifact: dict[str, Any]) -> None: + artifact["records"].sort( + key=lambda record: ( + record["scope_identity"], + record["root_instance_id"], + record["root_runtime_id"], + record["timer_id"], + ) + ) + artifact = seal_timer_records(artifact) + if not _validator("timer-record-v1.schema.json").is_valid(artifact): + raise TimerError("timer_capability_mismatch") + connection.execute( + "INSERT INTO determa_timer_helpers VALUES (?,?) " + "ON CONFLICT(scope_identity) DO UPDATE SET document=excluded.document", + (self.scope_identity, canonical_bytes(artifact)), + ) + + def _event(self, arguments: Mapping[str, Any]) -> None: + declaration = self.bundle.raw.get("events", {}).get(arguments["event_name"]) + try: + payload = decoded_typed_value(arguments["payload"]) + valid = ( + declaration + and declaration.get("direction") == "input" + and _normalize_payload(declaration, payload) is not None + ) + except (TypeError, ValueError, KeyError): + valid = False + if not valid: + raise TimerError("invalid_timer_request") + + def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, Any]: + if not isinstance(request, Mapping) or request.get("interface") != "determa.timer_helper": + return _result(None, None, "unsupported_timer_protocol") + if type(request.get("interface_version")) is not int or request["interface_version"] != 1: + return _result(None, None, "unsupported_timer_protocol_version") + request = copy.deepcopy(dict(request)) + # Normative invalid-time vectors retain recognized operation identities. + # Check the closed structure with a canonical time placeholder, then check + # the original time domain before any clock read or record write. + structural = copy.deepcopy(request) + if structural.get("operation") == "schedule" and isinstance( + structural.get("arguments"), dict + ): + for field in ("deadline_at", "delay_nanoseconds"): + if field in structural["arguments"]: + structural["arguments"][field] = "0" + if not _validator("timer-helper-operation-v1.schema.json", "request").is_valid( + structural + ) or request.get("request_digest") != timer_request_digest(request): + return _result(None, None, "invalid_timer_request") + if principal not in self.principals or ( + request["scope_identity"], + request["root_instance_id"], + request["root_runtime_id"], + ) != (self.scope_identity, self.root_instance_id, self.root_runtime_id): + return _result(request, None, "unauthorized_timer_scope") + connection = self._connect() + try: + self._schema(connection) + connection.execute("BEGIN IMMEDIATE") + artifact = self._load(connection) + record = next( + ( + record + for record in artifact["records"] + if record["timer_id"] == request["timer_id"] + and record["root_instance_id"] == self.root_instance_id + and record["root_runtime_id"] == self.root_runtime_id + ), + None, + ) + previous = next( + ( + receipt + for receipt in artifact["operation_receipts"] + if receipt["operation_id"] == request["operation_id"] + ), + None, + ) + if previous is not None: + return ( + copy.deepcopy(previous["result"]) + if previous["request_digest"] == request["request_digest"] + else _result(request, record, "timer_operation_conflict") + ) + operation = request["operation"] + if operation == "schedule": + if record is not None: + return _result(request, record, "timer_id_conflict") + self._event(request["arguments"]) + arguments = request["arguments"] + if "deadline_at" in arguments: + deadline = _time(arguments["deadline_at"]) + else: + delay = _time(arguments["delay_nanoseconds"], duration=True) + try: + now = _time(self.trusted_clock()) + except Exception as error: + raise TimerError("timer_clock_unavailable") from error + deadline = now + delay + if not -(2**63) <= deadline <= 2**63 - 1: + raise TimerError("timer_deadline_overflow") + record = { + key: request[key] + for key in ("scope_identity", "root_instance_id", "root_runtime_id", "timer_id") + } + record.update( + schedule_request_digest=request["request_digest"], + clock_basis="unix_nanoseconds", + deadline_at=str(deadline), + event_name=arguments["event_name"], + payload=arguments["payload"], + correlation_id=arguments["correlation_id"], + event_id=hash_value( + [ + "determa-timer-fire-event-1", + "1", + self.scope_identity, + self.root_instance_id, + self.root_runtime_id, + request["timer_id"], + ] + ), + state="pending", + revision="1", + attempt_fence="0", + worker_principal=None, + expires_at=None, + admission_receipt_digest=None, + ) + artifact["records"].append(record) + elif operation == "read_timer": + return ( + _result(request, record) + if record + else _result(request, None, "timer_not_found") + ) + elif operation == "cancel": + if record is None: + return _result(request, None, "timer_not_found") + if request["arguments"]["expected_revision"] != record["revision"]: + return _result(request, record, "timer_revision_conflict") + if record["state"] == "claimed": + return _result(request, record, "timer_fire_in_progress") + if record["state"] == "fired": + return _result(request, record, "timer_already_fired") + if record["state"] == "pending": + record.update(state="cancelled", revision=str(int(record["revision"]) + 1)) + else: + return _result(request, record, "timer_capability_mismatch") + result = _result(request, record) + artifact["operation_receipts"].append( + { + "operation_id": request["operation_id"], + "request_digest": request["request_digest"], + "result": result, + } + ) + self._persist(connection, artifact) + connection.commit() + return result + except TimerError as error: + return _result(request, None, error.code) + finally: + connection.rollback() + connection.close() diff --git a/tests/test_timers.py b/tests/test_timers.py new file mode 100644 index 0000000..5fe3067 --- /dev/null +++ b/tests/test_timers.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from determa.state import load_bundle +from determa.state.timers import SQLiteTimerHelper, timer_request_digest + +_ROOT_RUNTIME = "sha256:9b0e3238e782cb7d912febc1acc4e4c03a121d201764d0bc9658b969feac4dc8" + + +def bundle(): + return load_bundle( + { + "format": 1, + "namespace": "timer.tests", + "events": {"received": {"direction": "input"}}, + "machines": [ + { + "machine_id": "timer", + "root": { + "type": "composite", + "initial": {"transition_to": "waiting"}, + "states": {"waiting": {"on_events": {"received": {}}}}, + }, + } + ], + } + ) + + +def request(operation="schedule", operation_id="schedule-A", arguments=None): + value = { + "interface": "determa.timer_helper", + "interface_version": 1, + "scope_identity": "scope-archive-example", + "root_instance_id": "server-1", + "root_runtime_id": _ROOT_RUNTIME, + "timer_id": "timer-A", + "operation": operation, + "operation_id": operation_id, + "arguments": arguments + if arguments is not None + else { + "clock_basis": "unix_nanoseconds", + "delay_nanoseconds": "10", + "event_name": "received", + "payload": ["map", []], + "correlation_id": None, + }, + } + value["request_digest"] = timer_request_digest(value) + return value + + +def helper(path: Path, clock): + value = request() + return SQLiteTimerHelper( + path, + bundle(), + scope_identity=value["scope_identity"], + root_instance_id=value["root_instance_id"], + root_runtime_id=value["root_runtime_id"], + principals=frozenset({"operator"}), + trusted_clock=clock, + ) + + +def test_committed_schedule_survives_restart_and_replays_original_result_after_cancel(tmp_path): + calls = [] + initial = helper(tmp_path / "timer.sqlite", lambda: calls.append("clock") or "100") + initial.setup_schema() + scheduled = initial.execute(request(), principal="operator") + assert scheduled["status"] == "accepted" + assert initial.snapshot()["records"][0]["deadline_at"] == "110" + assert calls == ["clock"] + + def offline(): + raise AssertionError("retained replay/cancel/read must not access the clock") + + restarted = helper(tmp_path / "timer.sqlite", offline) + cancelled = restarted.execute( + request("cancel", "cancel-A", {"expected_revision": "1"}), principal="operator" + ) + assert cancelled["status"] == "accepted" and cancelled["record_revision"] == "2" + before = restarted.snapshot() + assert restarted.execute(request(), principal="operator") == scheduled + assert restarted.snapshot() == before + read = restarted.execute(request("read_timer", "read-A", {}), principal="operator") + assert read["record_revision"] == "2" and read["event_id"] == scheduled["event_id"] + assert restarted.snapshot() == before + + +@pytest.mark.parametrize( + "time", ["-0", "+1", "01", "1.5", "1e2", "9223372036854775808", "-1", 1, True] +) +def test_invalid_duration_changes_no_records_and_never_reads_clock(tmp_path, time): + def forbidden_clock(): + raise AssertionError("invalid time reached the clock") + + installed = helper(tmp_path / "timer.sqlite", forbidden_clock) + installed.setup_schema() + before = installed.snapshot() + value = request() + value["arguments"]["delay_nanoseconds"] = time + value["request_digest"] = timer_request_digest(value) + result = installed.execute(value, principal="operator") + assert result["error_code"] == "invalid_timer_time" + assert installed.snapshot() == before + + +def test_target_incarnation_and_principal_are_authorized_before_replay(tmp_path): + installed = helper(tmp_path / "timer.sqlite", lambda: "100") + installed.setup_schema() + assert installed.execute(request(), principal="operator")["status"] == "accepted" + before = installed.snapshot() + for value, principal in [(request(), "stranger"), (request(), "operator")]: + if principal == "operator": + value["root_runtime_id"] = "sha256:" + "0" * 64 + value["request_digest"] = timer_request_digest(value) + result = installed.execute(value, principal=principal) + assert result["error_code"] == "unauthorized_timer_scope" + assert result["record_revision"] is None and result["event_id"] is None + assert installed.snapshot() == before + + +def test_operation_conflict_preserves_exact_durable_schedule(tmp_path): + installed = helper(tmp_path / "timer.sqlite", lambda: "100") + installed.setup_schema() + value = request() + assert installed.execute(value, principal="operator")["status"] == "accepted" + before = installed.snapshot() + changed = copy.deepcopy(value) + changed["arguments"]["delay_nanoseconds"] = "20" + changed["request_digest"] = timer_request_digest(changed) + assert ( + installed.execute(changed, principal="operator")["error_code"] == "timer_operation_conflict" + ) + assert installed.snapshot() == before + + +def test_invalid_record_digest_refuses_read_after_restart(tmp_path): + import sqlite3 + + path = tmp_path / "timer.sqlite" + installed = helper(path, lambda: "100") + installed.setup_schema() + assert installed.execute(request(), principal="operator")["status"] == "accepted" + damaged = installed.snapshot() + damaged["records"][0]["deadline_at"] = "999" + with sqlite3.connect(path) as connection: + connection.execute( + "UPDATE determa_timer_helpers SET document=?", (json.dumps(damaged).encode(),) + ) + restarted = helper(path, lambda: "100") + result = restarted.execute(request("read_timer", "read-A", {}), principal="operator") + assert result["error_code"] == "timer_capability_mismatch" From 27c3fa8eab1c7f825536a1a8eeeb7c870c80f027 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 16:27:55 +0000 Subject: [PATCH 09/32] WIP timers: persist authenticated due claims and refuse ambiguous reclaim --- src/determa/state/timers.py | 91 ++++++++++++++++++++++++---- tests/test_timers.py | 116 ++++++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 13 deletions(-) diff --git a/src/determa/state/timers.py b/src/determa/state/timers.py index 1a0cc88..b8fb161 100644 --- a/src/determa/state/timers.py +++ b/src/determa/state/timers.py @@ -1,6 +1,6 @@ """Explicit foreground SQLite timer records, separate from portable checkpoints. -This implementation checkpoint supports durable schedule/cancel/read operations. +This implementation checkpoint supports durable schedule/cancel/read and initial claims. Fire, coordinated admission, verified provider installation and archive integration remain unfinished; no completed timer profile is advertised yet. """ @@ -18,7 +18,8 @@ from .definition import Bundle from .engine import _normalize_payload -from .wire import _schema_registry, canonical_bytes, decoded_typed_value, hash_value +from .errors import ArtifactError +from .wire import _schema_registry, canonical_bytes, decoded_typed_value, hash_value, strict_json _TIME = re.compile(r"(?:0|-?[1-9][0-9]*)\Z") _TABLE = ( @@ -93,7 +94,9 @@ def _result( if record and record["attempt_fence"] != "0" else None, "clock_basis": "unix_nanoseconds", - "expires_at": None, + "expires_at": record["expires_at"] + if code is None and request is not None and request["operation"] == "claim_fire" and record + else None, "event_id": record["event_id"] if record else None, "delivery_state": "admitted" if record and record["admission_receipt_digest"] else "none", "error_code": code, @@ -128,6 +131,8 @@ def __init__( root_runtime_id: str, principals: frozenset[str], trusted_clock: Callable[[], str], + worker_principals: frozenset[str] = frozenset(), + claim_lease_nanoseconds: str = "20", ) -> None: if ( str(path) == ":memory:" @@ -145,11 +150,20 @@ def __init__( self.root_runtime_id = root_runtime_id self.principals = frozenset(principals) self.trusted_clock = trusted_clock + self.worker_principals = frozenset(worker_principals) + self.claim_lease_nanoseconds = _time(claim_lease_nanoseconds, duration=True) + if self.claim_lease_nanoseconds == 0 or not self.worker_principals <= self.principals: + raise ValueError( + "workers must be authorized principals and the claim lease must be positive" + ) def _connect(self) -> sqlite3.Connection: connection = sqlite3.connect(self.path, isolation_level=None, timeout=30) - connection.execute("PRAGMA journal_mode=WAL") + journal = connection.execute("PRAGMA journal_mode=WAL").fetchone() connection.execute("PRAGMA synchronous=FULL") + if journal != ("wal",) or connection.execute("PRAGMA synchronous").fetchone() != (2,): + connection.close() + raise TimerError("timer_capability_mismatch") return connection def setup_schema(self) -> None: @@ -184,7 +198,13 @@ def _load(self, connection: sqlite3.Connection) -> dict[str, Any]: "operation_receipts": [], } ) - artifact: dict[str, Any] = json.loads(bytes(row[0])) + try: + parsed, raw = strict_json(bytes(row[0])) + if not isinstance(parsed, dict) or raw != canonical_bytes(parsed): + raise TimerError("timer_capability_mismatch") + artifact: dict[str, Any] = parsed + except (ArtifactError, TypeError, ValueError) as error: + raise TimerError("timer_capability_mismatch") from error if ( not _validator("timer-record-v1.schema.json").is_valid(artifact) or seal_timer_records(artifact) != artifact @@ -231,6 +251,12 @@ def _event(self, arguments: Mapping[str, Any]) -> None: if not valid: raise TimerError("invalid_timer_request") + def _now(self) -> int: + try: + return _time(self.trusted_clock()) + except Exception as error: + raise TimerError("timer_clock_unavailable") from error + def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, Any]: if not isinstance(request, Mapping) or request.get("interface") != "determa.timer_helper": return _result(None, None, "unsupported_timer_protocol") @@ -247,9 +273,13 @@ def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, An for field in ("deadline_at", "delay_nanoseconds"): if field in structural["arguments"]: structural["arguments"][field] = "0" - if not _validator("timer-helper-operation-v1.schema.json", "request").is_valid( - structural - ) or request.get("request_digest") != timer_request_digest(request): + try: + valid = _validator("timer-helper-operation-v1.schema.json", "request").is_valid( + structural + ) and request.get("request_digest") == timer_request_digest(request) + except (ArtifactError, TypeError, ValueError, OverflowError): + valid = False + if not valid: return _result(None, None, "invalid_timer_request") if principal not in self.principals or ( request["scope_identity"], @@ -257,6 +287,12 @@ def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, An request["root_runtime_id"], ) != (self.scope_identity, self.root_instance_id, self.root_runtime_id): return _result(request, None, "unauthorized_timer_scope") + if request["operation"] in ("claim_fire", "complete_fire") and ( + principal not in self.worker_principals + or request["arguments"]["worker_principal"] != principal + ): + return _result(request, None, "unauthorized_timer_scope") + record = None connection = self._connect() try: self._schema(connection) @@ -296,10 +332,7 @@ def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, An deadline = _time(arguments["deadline_at"]) else: delay = _time(arguments["delay_nanoseconds"], duration=True) - try: - now = _time(self.trusted_clock()) - except Exception as error: - raise TimerError("timer_clock_unavailable") from error + now = self._now() deadline = now + delay if not -(2**63) <= deadline <= 2**63 - 1: raise TimerError("timer_deadline_overflow") @@ -349,6 +382,38 @@ def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, An return _result(request, record, "timer_already_fired") if record["state"] == "pending": record.update(state="cancelled", revision=str(int(record["revision"]) + 1)) + elif operation == "claim_fire": + if record is None: + return _result(request, None, "timer_not_found") + if request["arguments"]["expected_revision"] != record["revision"]: + return _result(request, record, "timer_revision_conflict") + if record["state"] == "fired": + return _result(request, record, "timer_already_fired") + if record["state"] == "cancelled": + return _result(request, record, "timer_cancelled") + now = self._now() + if record["state"] == "claimed": + # A timeout is not proof that a previous admission did not commit. + # Reclaim is refused until coordinated fate verification is implemented. + return _result( + request, + record, + "delivery_ambiguous" + if now >= _time(record["expires_at"]) + else "timer_fire_in_progress", + ) + if now < _time(record["deadline_at"]): + return _result(request, record, "timer_not_due") + expires_at = now + self.claim_lease_nanoseconds + if expires_at > 2**63 - 1: + return _result(request, record, "timer_deadline_overflow") + record.update( + state="claimed", + revision=str(int(record["revision"]) + 1), + attempt_fence=str(int(record["attempt_fence"]) + 1), + worker_principal=principal, + expires_at=str(expires_at), + ) else: return _result(request, record, "timer_capability_mismatch") result = _result(request, record) @@ -363,7 +428,7 @@ def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, An connection.commit() return result except TimerError as error: - return _result(request, None, error.code) + return _result(request, record, error.code) finally: connection.rollback() connection.close() diff --git a/tests/test_timers.py b/tests/test_timers.py index 5fe3067..476cc04 100644 --- a/tests/test_timers.py +++ b/tests/test_timers.py @@ -158,3 +158,119 @@ def test_invalid_record_digest_refuses_read_after_restart(tmp_path): restarted = helper(path, lambda: "100") result = restarted.execute(request("read_timer", "read-A", {}), principal="operator") assert result["error_code"] == "timer_capability_mismatch" + + +def worker_helper(path, clock): + return SQLiteTimerHelper( + path, + bundle(), + scope_identity="scope-archive-example", + root_instance_id="server-1", + root_runtime_id=_ROOT_RUNTIME, + principals=frozenset({"operator", "worker-A", "worker-B"}), + worker_principals=frozenset({"worker-A", "worker-B"}), + trusted_clock=clock, + ) + + +def test_due_claim_is_durable_and_replay_does_not_recheck_clock_or_revision(tmp_path): + path = tmp_path / "timers.sqlite" + now = ["100"] + installed = worker_helper(path, lambda: now[0]) + installed.setup_schema() + installed.execute(request(), principal="operator") + claim = request( + "claim_fire", + "claim-A", + { + "expected_revision": "1", + "clock_basis": "unix_nanoseconds", + "worker_principal": "worker-A", + }, + ) + before = installed.snapshot() + now[0] = "109" + assert installed.execute(claim, principal="worker-A")["error_code"] == "timer_not_due" + assert installed.snapshot() == before + now[0] = "110" + result = installed.execute(claim, principal="worker-A") + assert result["status"] == "accepted" and result["attempt_fence"] == "1" + assert result["record_revision"] == "2" and result["expires_at"] == "130" + committed = installed.snapshot() + + def offline(): + raise AssertionError("retained claim replay read the clock") + + restarted = worker_helper(path, offline) + assert restarted.execute(claim, principal="worker-A") == result + assert restarted.snapshot() == committed + assert ( + restarted.execute( + request("cancel", "cancel-A", {"expected_revision": "2"}), principal="operator" + )["error_code"] + == "timer_fire_in_progress" + ) + assert restarted.snapshot() == committed + + +def test_expired_claim_without_commit_fate_proof_never_grants_a_new_fence(tmp_path): + path = tmp_path / "timers.sqlite" + now = ["100"] + installed = worker_helper(path, lambda: now[0]) + installed.setup_schema() + installed.execute(request(), principal="operator") + now[0] = "110" + claim = request( + "claim_fire", + "claim-A", + { + "expected_revision": "1", + "clock_basis": "unix_nanoseconds", + "worker_principal": "worker-A", + }, + ) + assert installed.execute(claim, principal="worker-A")["status"] == "accepted" + before = installed.snapshot() + retry = request( + "claim_fire", + "retry-B", + { + "expected_revision": "2", + "clock_basis": "unix_nanoseconds", + "worker_principal": "worker-B", + }, + ) + now[0] = "131" + refused = worker_helper(path, lambda: now[0]).execute(retry, principal="worker-B") + assert refused["error_code"] == "delivery_ambiguous" and refused["attempt_fence"] == "1" + assert installed.snapshot() == before + assert ( + installed.execute(retry, principal="worker-A")["error_code"] == "unauthorized_timer_scope" + ) + assert installed.snapshot() == before + + +@pytest.mark.parametrize("encoding", ["duplicate", "noncanonical", "invalid"]) +def test_stored_artifact_requires_strict_canonical_json(tmp_path, encoding): + import sqlite3 + + from determa.state.wire import canonical_bytes + + path = tmp_path / "timers.sqlite" + installed = helper(path, lambda: "100") + installed.setup_schema() + installed.execute(request(), principal="operator") + document = installed.snapshot() + raw = canonical_bytes(document) + if encoding == "duplicate": + raw = b'{"records":[],' + raw[1:] + elif encoding == "noncanonical": + raw = json.dumps(document, indent=2).encode() + else: + raw = b"invalid" + with sqlite3.connect(path) as connection: + connection.execute("UPDATE determa_timer_helpers SET document=?", (raw,)) + assert ( + installed.execute(request("read_timer", "read-A", {}), principal="operator")["error_code"] + == "timer_capability_mismatch" + ) From 20457d770337f8174a39ef9d7ee3efd9f29f99ef Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 16:35:14 +0000 Subject: [PATCH 10/32] WIP timers: coordinate firing and actual admission in host-owned SQLite transaction --- src/determa/state/host.py | 12 ++ src/determa/state/stores/sqlite.py | 95 ++++++++++ src/determa/state/timers.py | 196 +++++++++++++++++++-- tests/test_sqlite_shared_transaction.py | 98 +++++++++++ tests/test_timer_coordinated_admission.py | 200 ++++++++++++++++++++++ 5 files changed, 591 insertions(+), 10 deletions(-) create mode 100644 tests/test_sqlite_shared_transaction.py create mode 100644 tests/test_timer_coordinated_admission.py diff --git a/src/determa/state/host.py b/src/determa/state/host.py index 6a362c9..369c65d 100644 --- a/src/determa/state/host.py +++ b/src/determa/state/host.py @@ -1526,6 +1526,7 @@ def __init__(self, host: ExecutionHost, root_instance_id: str) -> None: self.root_instance_id = root_instance_id self._active = True self._response: dict[str, Any] | None = None + self._operation: str | None = None def _stage( self, @@ -1537,11 +1538,16 @@ def _stage( if self._response is not None: raise ExecutionHostError("shared_transaction_operation_conflict") self._response = invoke() + self._operation = operation return StagedExecutionResult(operation) def _finish(self) -> dict[str, Any]: if self._response is None: raise ExecutionHostError("shared_transaction_operation_required") + # Queue-bearing admission returns committed acceptance evidence rather + # than the result discriminator used by create/migration operations. + if self._operation == "admit_v1" and set(self._response) == {"evidence"}: + return self._response if self._response["result"] not in { "committed", "pending", @@ -1554,6 +1560,12 @@ def _finish(self) -> dict[str, Any]: def _deactivate(self) -> None: self._active = False + def read_checkpoint(self) -> RestoredExecutionCheckpoint | None: + """Read and resolver-validate the bound root inside this transaction.""" + if not self._active: + raise ExecutionHostError("shared_transaction_closed") + return self._host.read_checkpoint(self.root_instance_id) + def create_v1( self, bundle: Bundle | BundleSource, diff --git a/src/determa/state/stores/sqlite.py b/src/determa/state/stores/sqlite.py index 45f684d..734c5ec 100644 --- a/src/determa/state/stores/sqlite.py +++ b/src/determa/state/stores/sqlite.py @@ -17,6 +17,7 @@ PERMANENT_OUTBOX_TERMINAL_RETENTION, PERMANENT_RECEIPT_RETENTION, ROOT_IDENTITY_RETENTION, + SHARED_APPLICATION_TRANSACTION, ExecutionStore, ExecutionStoreError, ExecutionStoreTransaction, @@ -106,6 +107,68 @@ def replace( return cursor.rowcount == 1 +class SQLiteApplicationCursor: + """Rows from host-owned SQL without exposing the connection's commit controls.""" + + def __init__(self, cursor: sqlite3.Cursor) -> None: + self._cursor = cursor + + @property + def rowcount(self) -> int: + return self._cursor.rowcount + + def fetchone(self) -> Any: + return self._cursor.fetchone() + + def fetchall(self) -> list[Any]: + return self._cursor.fetchall() + + +class SQLiteApplicationTransaction: + """Foreground SQL resource whose lifetime and commit belong to ExecutionHost. + + Application callbacks receive statement execution and rows, not a second + transaction API. SQL transaction control, attached databases and PRAGMA changes + are refused. The resource is closed before the host commits or rolls back. + """ + + def __init__(self, connection: sqlite3.Connection) -> None: + self._connection = connection + self._active = True + + def execute(self, statement: str, parameters: Any = ()) -> SQLiteApplicationCursor: + if not self._active: + raise ExecutionStoreError("shared_transaction_closed") + return SQLiteApplicationCursor(self._connection.execute(statement, parameters)) + + def _close(self) -> None: + self._active = False + + +def _application_authorizer( + operation: int, + first: str | None, + second: str | None, + database: str | None, + trigger: str | None, +) -> int: + del database, trigger + if operation == sqlite3.SQLITE_PRAGMA and first == "integrity_check" and second is None: + return sqlite3.SQLITE_OK + return ( + sqlite3.SQLITE_DENY + if operation + in ( + sqlite3.SQLITE_TRANSACTION, + sqlite3.SQLITE_SAVEPOINT, + sqlite3.SQLITE_ATTACH, + sqlite3.SQLITE_DETACH, + sqlite3.SQLITE_PRAGMA, + ) + else sqlite3.SQLITE_OK + ) + + class SQLiteExecutionStore(ExecutionStore): """Single-writer durable SQLite storage under verified PRAGMA settings.""" @@ -118,6 +181,7 @@ def __init__( timeout: float = 30.0, replay_retention: str = "bounded", outbox_retention: str = "none", + shared_application_transactions: bool = False, ) -> None: self.path = str(path) self.journal_mode = journal_mode.upper() @@ -125,6 +189,7 @@ def __init__( self.timeout = timeout self.replay_retention = replay_retention self.outbox_retention = outbox_retention + self.shared_application_transactions = shared_application_transactions if ( not self.path or self.path == ":memory:" @@ -133,6 +198,7 @@ def __init__( or timeout <= 0 or replay_retention not in _REPLAY_RETENTION_MODES or outbox_retention not in _OUTBOX_RETENTION_MODES + or type(shared_application_transactions) is not bool ): raise ExecutionStoreError(AdapterCode.INVALID_ADAPTER_CONFIGURATION) @@ -142,6 +208,8 @@ def capabilities(self) -> frozenset[str]: if not self._policy_is_valid(): return frozenset(capabilities) capabilities.add(ROOT_IDENTITY_RETENTION) + if self.shared_application_transactions: + capabilities.add(SHARED_APPLICATION_TRANSACTION) if self.replay_retention == "permanent": capabilities.add(PERMANENT_RECEIPT_RETENTION) if self.outbox_retention == "strict": @@ -416,6 +484,33 @@ def setup_schema(self) -> None: finally: connection.close() + @contextmanager + def shared_transaction( + self, + root_instance_id: str, + ) -> Iterator[tuple[SQLiteApplicationTransaction, ExecutionStoreTransaction]]: + if not self.shared_application_transactions: + raise ExecutionStoreError(AdapterCode.ADAPTER_CAPABILITY_MISMATCH) + connection = self._connect() + application = SQLiteApplicationTransaction(connection) + try: + self._validate_schema(connection) + connection.execute("BEGIN IMMEDIATE") + connection.set_authorizer(_application_authorizer) + yield application, _SQLiteTransaction(connection, root_instance_id) + application._close() + connection.set_authorizer(None) + if not connection.in_transaction: + raise ExecutionStoreError(AdapterCode.ADAPTER_CAPABILITY_MISMATCH) + connection.commit() + except BaseException: + application._close() + connection.set_authorizer(None) + connection.rollback() + raise + finally: + connection.close() + def health(self) -> Mapping[str, Any]: try: connection = self._connect() diff --git a/src/determa/state/timers.py b/src/determa/state/timers.py index b8fb161..0ac920b 100644 --- a/src/determa/state/timers.py +++ b/src/determa/state/timers.py @@ -1,8 +1,8 @@ """Explicit foreground SQLite timer records, separate from portable checkpoints. -This implementation checkpoint supports durable schedule/cancel/read and initial claims. -Fire, coordinated admission, verified provider installation and archive integration -remain unfinished; no completed timer profile is advertised yet. +This implementation checkpoint supports durable schedule/cancel/read, initial claims +and coordinated SQLite admission. Proved-fate reclaim, verified provider installation, +archive and recovery integration remain unfinished; no completed profile is advertised. """ from __future__ import annotations @@ -19,6 +19,13 @@ from .definition import Bundle from .engine import _normalize_payload from .errors import ArtifactError +from .host import ( + ExecutionHost, + ExecutionHostError, + SharedExecutionTransaction, + delivery_request_digest, +) +from .stores.sqlite import SQLiteApplicationTransaction, SQLiteExecutionStore from .wire import _schema_registry, canonical_bytes, decoded_typed_value, hash_value, strict_json _TIME = re.compile(r"(?:0|-?[1-9][0-9]*)\Z") @@ -42,6 +49,13 @@ def __init__(self, code: str) -> None: super().__init__(code) +class _TimerReturn(Exception): + """Return a replay/refusal only after rolling back the shared host callback.""" + + def __init__(self, result: dict[str, Any]) -> None: + self.result = result + + def timer_request_digest(request: Mapping[str, Any]) -> str: return hash_value( [ @@ -133,6 +147,7 @@ def __init__( trusted_clock: Callable[[], str], worker_principals: frozenset[str] = frozenset(), claim_lease_nanoseconds: str = "20", + coordinated_host: ExecutionHost | None = None, ) -> None: if ( str(path) == ":memory:" @@ -150,6 +165,13 @@ def __init__( self.root_runtime_id = root_runtime_id self.principals = frozenset(principals) self.trusted_clock = trusted_clock + self.coordinated_host = coordinated_host + if coordinated_host is not None and ( + type(coordinated_host.store) is not SQLiteExecutionStore + or Path(coordinated_host.store.path).resolve() != Path(self.path) + or not coordinated_host.store.shared_application_transactions + ): + raise ValueError("coordinated admission requires the same host-owned SQLite database") self.worker_principals = frozenset(worker_principals) self.claim_lease_nanoseconds = _time(claim_lease_nanoseconds, duration=True) if self.claim_lease_nanoseconds == 0 or not self.worker_principals <= self.principals: @@ -171,20 +193,23 @@ def setup_schema(self) -> None: connection.execute(_TABLE.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1)) self.validate_schema() - def _schema(self, connection: sqlite3.Connection) -> None: + def _schema(self, connection: sqlite3.Connection | SQLiteApplicationTransaction) -> None: rows = connection.execute( - "SELECT sql FROM sqlite_master WHERE type='table' AND name='determa_timer_helpers'" + "SELECT type,sql FROM sqlite_master " + "WHERE tbl_name='determa_timer_helpers' ORDER BY type" ).fetchall() - if rows != [(_TABLE,)] or connection.execute("PRAGMA integrity_check").fetchone() != ( - "ok", - ): + if rows != [("index", None), ("table", _TABLE)] or connection.execute( + "PRAGMA integrity_check" + ).fetchone() != ("ok",): raise TimerError("timer_capability_mismatch") def validate_schema(self) -> None: with closing(self._connect()) as connection: self._schema(connection) - def _load(self, connection: sqlite3.Connection) -> dict[str, Any]: + def _load( + self, connection: sqlite3.Connection | SQLiteApplicationTransaction + ) -> dict[str, Any]: row = connection.execute( "SELECT document FROM determa_timer_helpers WHERE scope_identity=?", (self.scope_identity,), @@ -219,7 +244,11 @@ def snapshot(self) -> dict[str, Any]: self._schema(connection) return self._load(connection) - def _persist(self, connection: sqlite3.Connection, artifact: dict[str, Any]) -> None: + def _persist( + self, + connection: sqlite3.Connection | SQLiteApplicationTransaction, + artifact: dict[str, Any], + ) -> None: artifact["records"].sort( key=lambda record: ( record["scope_identity"], @@ -257,6 +286,151 @@ def _now(self) -> int: except Exception as error: raise TimerError("timer_clock_unavailable") from error + def _complete(self, request: Mapping[str, Any], principal: str) -> dict[str, Any]: + host = self.coordinated_host + assert host is not None + if ( + type(host.store) is not SQLiteExecutionStore + or Path(host.store.path).resolve() != Path(self.path) + or not host.store.shared_application_transactions + ): + return _result(request, None, "timer_capability_mismatch") + committed: list[dict[str, Any]] = [] + + def callback( + sql: SQLiteApplicationTransaction, execution: SharedExecutionTransaction + ) -> None: + record = None + try: + self._schema(sql) + artifact = self._load(sql) + record = next( + ( + item + for item in artifact["records"] + if item["timer_id"] == request["timer_id"] + and item["root_instance_id"] == self.root_instance_id + and item["root_runtime_id"] == self.root_runtime_id + ), + None, + ) + previous = next( + ( + item + for item in artifact["operation_receipts"] + if item["operation_id"] == request["operation_id"] + ), + None, + ) + if previous is not None: + raise _TimerReturn( + copy.deepcopy(previous["result"]) + if previous["request_digest"] == request["request_digest"] + else _result(request, record, "timer_operation_conflict") + ) + if record is None: + raise TimerError("timer_not_found") + arguments = request["arguments"] + if record["attempt_fence"] != arguments["attempt_fence"]: + raise TimerError("timer_stale_fence") + if record["revision"] != arguments["expected_revision"]: + raise TimerError("timer_revision_conflict") + if record["state"] != "claimed": + raise TimerError( + "timer_already_fired" + if record["state"] == "fired" + else "timer_cancelled" + if record["state"] == "cancelled" + else "timer_stale_fence" + ) + if record["worker_principal"] != principal: + raise TimerError("timer_worker_mismatch") + if record["event_id"] != arguments["event_id"]: + raise TimerError("timer_event_conflict") + if self._now() >= _time(record["expires_at"]): + raise TimerError("timer_stale_fence") + restored = execution.read_checkpoint() + if restored is None: + raise TimerError("timer_admission_rejected") + before = restored.document + aggregate = before["root_record"].get("aggregate_state") + if ( + not aggregate + or aggregate["root_runtime_id"] != self.root_runtime_id + or (aggregate["validated_bundle_fingerprint"] != self.bundle.fingerprint) + ): + raise TimerError("unauthorized_timer_scope") + envelope = { + "event": record["event_name"], + "event_id": record["event_id"], + "cause_id": record["event_id"], + "source": {"host": True}, + "target": { + "root": { + "root_instance_id": self.root_instance_id, + "root_runtime_id": self.root_runtime_id, + } + }, + "payload": record["payload"], + } + if record["correlation_id"] is not None: + envelope["correlation_id"] = record["correlation_id"] + envelope_digest = delivery_request_digest(self.root_instance_id, "input", envelope) + execution.admit_v1( + [ + { + "delivery_mode": "input", + "envelope": envelope, + "envelope_digest": envelope_digest, + } + ], + expected_revision=before["revision"], + expected_checkpoint_digest=before["execution_checkpoint_digest"], + ) + candidate = execution.read_checkpoint() + assert candidate is not None + receipt = next( + ( + item + for item in candidate.document["operation_receipts"] + if item["operation_kind"] == "acceptance" + and item["event_id"] == record["event_id"] + ), + None, + ) + if receipt is None or receipt["request_digest"] != envelope_digest: + raise TimerError("timer_admission_rejected") + receipt_digest = hash_value(["determa-timer-admission-receipt-1", receipt]) + if receipt_digest != arguments["admission_receipt_digest"]: + raise TimerError("timer_event_conflict") + record.update( + state="fired", + revision=str(int(record["revision"]) + 1), + worker_principal=None, + expires_at=None, + admission_receipt_digest=receipt_digest, + ) + result = _result(request, record) + artifact["operation_receipts"].append( + { + "operation_id": request["operation_id"], + "request_digest": request["request_digest"], + "result": result, + } + ) + self._persist(sql, artifact) + committed.append(result) + except TimerError as error: + raise _TimerReturn(_result(request, record, error.code)) from error + except ExecutionHostError as error: + raise _TimerReturn(_result(request, record, "timer_admission_rejected")) from error + + try: + host.run_shared_transaction(self.root_instance_id, callback) + except _TimerReturn as returned: + return returned.result + return committed[0] + def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, Any]: if not isinstance(request, Mapping) or request.get("interface") != "determa.timer_helper": return _result(None, None, "unsupported_timer_protocol") @@ -292,6 +466,8 @@ def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, An or request["arguments"]["worker_principal"] != principal ): return _result(request, None, "unauthorized_timer_scope") + if request["operation"] == "complete_fire" and self.coordinated_host is not None: + return self._complete(request, principal) record = None connection = self._connect() try: diff --git a/tests/test_sqlite_shared_transaction.py b/tests/test_sqlite_shared_transaction.py new file mode 100644 index 0000000..127d859 --- /dev/null +++ b/tests/test_sqlite_shared_transaction.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import sqlite3 + +import pytest + +from determa.state import ExecutionHost, ExecutionStoreError, SQLiteExecutionStore, load_bundle +from determa.state.stores.base import SHARED_APPLICATION_TRANSACTION + +from .test_execution_stores import MACHINE, _resolver + + +def configured(tmp_path): + store = SQLiteExecutionStore(tmp_path / "shared.sqlite", shared_application_transactions=True) + store.setup_schema() + with sqlite3.connect(store.path) as connection: + connection.execute("CREATE TABLE applications (root TEXT PRIMARY KEY)") + return store, ExecutionHost(store, _resolver()) + + +def test_sqlite_shared_transaction_is_explicit_and_commits_application_and_root(tmp_path): + store, host = configured(tmp_path) + assert SHARED_APPLICATION_TRANSACTION in store.capabilities + assert SHARED_APPLICATION_TRANSACTION not in SQLiteExecutionStore(store.path).capabilities + retained = [] + + def callback(sql, execution): + retained.append(sql) + sql.execute("INSERT INTO applications VALUES (?)", ("root",)) + execution.create_v1(load_bundle(MACHINE), "counter", "create", {}) + # Another reader sees neither uncommitted write. + with sqlite3.connect(store.path) as connection: + assert connection.execute("SELECT root FROM applications").fetchall() == [] + assert ( + connection.execute( + "SELECT root_instance_id FROM determa_execution_checkpoints" + ).fetchall() + == [] + ) + + response = host.run_shared_transaction("root", callback) + assert response["result"] == "committed" + with sqlite3.connect(store.path) as connection: + assert connection.execute("SELECT root FROM applications").fetchall() == [("root",)] + assert ( + ExecutionHost(SQLiteExecutionStore(store.path), _resolver()).read_checkpoint("root") + is not None + ) + with pytest.raises(ExecutionStoreError, match="shared_transaction_closed"): + retained[0].execute("SELECT 1") + + +def test_sqlite_shared_callback_failure_rolls_back_both_writes(tmp_path): + store, host = configured(tmp_path) + + def callback(sql, execution): + sql.execute("INSERT INTO applications VALUES (?)", ("root",)) + execution.create_v1(load_bundle(MACHINE), "counter", "create", {}) + raise RuntimeError("application refused") + + with pytest.raises(RuntimeError, match="application refused"): + host.run_shared_transaction("root", callback) + assert host.read_checkpoint("root") is None + with sqlite3.connect(store.path) as connection: + assert connection.execute("SELECT root FROM applications").fetchall() == [] + + +@pytest.mark.parametrize( + "statement", + [ + "COMMIT", + "ROLLBACK", + "SAVEPOINT early", + "PRAGMA synchronous=OFF", + "ATTACH ':memory:' AS other", + ], +) +def test_sqlite_application_callback_cannot_commit_or_change_durability(tmp_path, statement): + store, host = configured(tmp_path) + + def callback(sql, execution): + sql.execute("INSERT INTO applications VALUES (?)", ("root",)) + execution.create_v1(load_bundle(MACHINE), "counter", "create", {}) + sql.execute(statement) + + with pytest.raises(sqlite3.DatabaseError, match="not authorized"): + host.run_shared_transaction("root", callback) + assert host.read_checkpoint("root") is None + with sqlite3.connect(store.path) as connection: + assert connection.execute("SELECT root FROM applications").fetchall() == [] + + +def test_sqlite_default_store_refuses_shared_transaction(tmp_path): + store = SQLiteExecutionStore(tmp_path / "default.sqlite") + store.setup_schema() + host = ExecutionHost(store, _resolver()) + with pytest.raises(Exception, match="adapter_capability_mismatch"): + host.run_shared_transaction("root", lambda sql, execution: None) diff --git a/tests/test_timer_coordinated_admission.py b/tests/test_timer_coordinated_admission.py new file mode 100644 index 0000000..090247d --- /dev/null +++ b/tests/test_timer_coordinated_admission.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +import copy + +import pytest + +from determa.state import ExecutionHost, MemoryArtifactResolver, SQLiteExecutionStore +from determa.state.checkpoint_v1 import admit_checkpoint_v1 +from determa.state.host import delivery_request_digest +from determa.state.timers import SQLiteTimerHelper, timer_request_digest +from determa.state.wire import hash_value + +from .test_timers import bundle, request + + +def configured(tmp_path): + path = tmp_path / "coordinated.sqlite" + definition = bundle() + resolver = MemoryArtifactResolver(definitions={definition.fingerprint: definition}) + store = SQLiteExecutionStore( + path, replay_retention="permanent", shared_application_transactions=True + ) + store.setup_schema() + host = ExecutionHost(store, resolver) + host.create_v1(definition, "timer", "server-1", "create-timer", {}) + checkpoint = host.read_checkpoint("server-1").document + runtime = checkpoint["root_record"]["aggregate_state"]["root_runtime_id"] + now = ["100"] + clock_calls = [] + + def clock(): + clock_calls.append(now[0]) + return now[0] + + helper = SQLiteTimerHelper( + path, + definition, + scope_identity="scope-archive-example", + root_instance_id="server-1", + root_runtime_id=runtime, + principals=frozenset({"operator", "worker-A"}), + worker_principals=frozenset({"worker-A"}), + trusted_clock=clock, + coordinated_host=host, + ) + helper.setup_schema() + + def command(operation, identifier, arguments=None): + value = request(operation, identifier, arguments) + value["root_runtime_id"] = runtime + value["request_digest"] = timer_request_digest(value) + return value + + assert ( + helper.execute(command("schedule", "schedule-A"), principal="operator")["status"] + == "accepted" + ) + now[0] = "110" + assert ( + helper.execute( + command( + "claim_fire", + "claim-A", + { + "expected_revision": "1", + "clock_basis": "unix_nanoseconds", + "worker_principal": "worker-A", + }, + ), + principal="worker-A", + )["status"] + == "accepted" + ) + record = helper.snapshot()["records"][0] + envelope = { + "event": "received", + "event_id": record["event_id"], + "cause_id": record["event_id"], + "source": {"host": True}, + "target": {"root": {"root_instance_id": "server-1", "root_runtime_id": runtime}}, + "payload": ["map", []], + } + delivery = { + "delivery_mode": "input", + "envelope": envelope, + "envelope_digest": delivery_request_digest("server-1", "input", envelope), + } + expected = admit_checkpoint_v1( + checkpoint, + [delivery], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + receipt = expected["operation_receipts"][-1] + complete = command( + "complete_fire", + "complete-A", + { + "expected_revision": "2", + "attempt_fence": "1", + "worker_principal": "worker-A", + "event_id": record["event_id"], + "admission_receipt_digest": hash_value(["determa-timer-admission-receipt-1", receipt]), + }, + ) + now[0] = "120" + return helper, host, complete, expected, now, clock_calls + + +def test_actual_admission_and_timer_fire_commit_together_and_replay_after_restart(tmp_path): + helper, host, complete, expected, now, calls = configured(tmp_path) + result = helper.execute(complete, principal="worker-A") + assert result["status"] == "accepted" and result["delivery_state"] == "admitted" + assert result["record_revision"] == "3" and result["expires_at"] is None + assert host.read_checkpoint("server-1").document == expected + committed = helper.snapshot() + assert ( + committed["records"][0]["admission_receipt_digest"] + == complete["arguments"]["admission_receipt_digest"] + ) + + def offline(): + raise AssertionError("retained fire replay read the clock") + + restarted_host = ExecutionHost( + SQLiteExecutionStore( + helper.path, replay_retention="permanent", shared_application_transactions=True + ), + host.artifact_resolver, + ) + restarted = SQLiteTimerHelper( + helper.path, + helper.bundle, + scope_identity=helper.scope_identity, + root_instance_id=helper.root_instance_id, + root_runtime_id=helper.root_runtime_id, + principals=helper.principals, + worker_principals=helper.worker_principals, + trusted_clock=offline, + coordinated_host=restarted_host, + ) + assert restarted.execute(complete, principal="worker-A") == result + assert restarted.snapshot() == committed + assert restarted_host.read_checkpoint("server-1").document == expected + assert calls == ["100", "110", "120"] + + +@pytest.mark.parametrize( + "field,value,code", + [ + ("attempt_fence", "0", "timer_stale_fence"), + ("event_id", "sha256:" + "0" * 64, "timer_event_conflict"), + ("admission_receipt_digest", "sha256:" + "0" * 64, "timer_event_conflict"), + ], +) +def test_false_fire_identity_or_receipt_rolls_back_actual_admission(tmp_path, field, value, code): + helper, host, complete, expected, now, calls = configured(tmp_path) + before_helper = helper.snapshot() + before_checkpoint = host.read_checkpoint("server-1").document + changed = copy.deepcopy(complete) + changed["arguments"][field] = value + changed["request_digest"] = timer_request_digest(changed) + assert helper.execute(changed, principal="worker-A")["error_code"] == code + assert helper.snapshot() == before_helper + assert host.read_checkpoint("server-1").document == before_checkpoint + + +def test_failure_after_staging_timer_and_actual_acceptance_rolls_back_both(tmp_path, monkeypatch): + helper, host, complete, expected, now, calls = configured(tmp_path) + before_helper = helper.snapshot() + before_checkpoint = host.read_checkpoint("server-1").document + persist = helper._persist + + def fail_after_write(sql, artifact): + persist(sql, artifact) + raise RuntimeError("native commit interrupted") + + monkeypatch.setattr(helper, "_persist", fail_after_write) + with pytest.raises(RuntimeError, match="native commit interrupted"): + helper.execute(complete, principal="worker-A") + assert helper.snapshot() == before_helper + assert host.read_checkpoint("server-1").document == before_checkpoint + + +def test_commit_before_response_failure_is_reconciled_from_retained_fire(tmp_path): + helper, host, complete, expected, now, calls = configured(tmp_path) + + def interrupted(boundary): + if boundary == "after_commit_before_response": + raise RuntimeError("lost response") + + host.fault_injector = interrupted + with pytest.raises(RuntimeError, match="lost response"): + helper.execute(complete, principal="worker-A") + assert helper.snapshot()["records"][0]["state"] == "fired" + assert host.read_checkpoint("server-1").document == expected + # Retained replay aborts the read-only callback before the response fault hook. + assert helper.execute(complete, principal="worker-A")["status"] == "accepted" + assert calls == ["100", "110", "120"] From 7ae17a20c6987406d39ce62a116daf7008be6bea Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 16:39:07 +0000 Subject: [PATCH 11/32] WIP timers: isolate protected storage and recheck lease before native commit --- src/determa/state/stores/sqlite.py | 54 ++++++++++++++++++++--- src/determa/state/timers.py | 45 +++++++++++++++---- tests/test_sqlite_shared_transaction.py | 28 ++++++++++++ tests/test_timer_coordinated_admission.py | 31 ++++++++++++- 4 files changed, 141 insertions(+), 17 deletions(-) diff --git a/src/determa/state/stores/sqlite.py b/src/determa/state/stores/sqlite.py index 734c5ec..6d386a0 100644 --- a/src/determa/state/stores/sqlite.py +++ b/src/determa/state/stores/sqlite.py @@ -4,7 +4,7 @@ import re import sqlite3 -from collections.abc import Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping from contextlib import contextmanager from pathlib import Path from typing import Any @@ -46,16 +46,28 @@ def _schema_tokens(source: str) -> list[str]: class _SQLiteTransaction(ExecutionStoreTransaction): - def __init__(self, connection: sqlite3.Connection, root_instance_id: str) -> None: + def __init__( + self, + connection: sqlite3.Connection, + root_instance_id: str, + *, + application_sql: bool = False, + ) -> None: self._connection = connection self._root_instance_id = root_instance_id + self._application_sql = application_sql + + def _execute(self, statement: str, parameters: Any = ()) -> sqlite3.Cursor: + if self._application_sql: + return _execute_host_owned(self._connection, statement, parameters) + return self._connection.execute(statement, parameters) @property def root_instance_id(self) -> str: return self._root_instance_id def load(self) -> bytes | None: - row = self._connection.execute( + row = self._execute( f"SELECT checkpoint FROM {_TABLE} WHERE root_instance_id = ?", (self._root_instance_id,), ).fetchone() @@ -66,7 +78,7 @@ def insert(self, checkpoint: bytes) -> bool: if root_instance_id != self._root_instance_id: raise ExecutionStoreError("transaction_root_mismatch") try: - self._connection.execute( + self._execute( f""" INSERT INTO {_TABLE} (root_instance_id, revision, checkpoint_digest, checkpoint) @@ -87,7 +99,7 @@ def replace( root_instance_id, revision, digest = checkpoint_metadata(checkpoint) if root_instance_id != self._root_instance_id: raise ExecutionStoreError("transaction_root_mismatch") - cursor = self._connection.execute( + cursor = self._execute( f""" UPDATE {_TABLE} SET revision = ?, checkpoint_digest = ?, checkpoint = ? @@ -135,6 +147,7 @@ class SQLiteApplicationTransaction: def __init__(self, connection: sqlite3.Connection) -> None: self._connection = connection self._active = True + self._commit_checks: list[Callable[[], None]] = [] def execute(self, statement: str, parameters: Any = ()) -> SQLiteApplicationCursor: if not self._active: @@ -144,6 +157,27 @@ def execute(self, statement: str, parameters: Any = ()) -> SQLiteApplicationCurs def _close(self) -> None: self._active = False + def _execute_host_owned(self, statement: str, parameters: Any = ()) -> SQLiteApplicationCursor: + if not self._active: + raise ExecutionStoreError("shared_transaction_closed") + return SQLiteApplicationCursor(_execute_host_owned(self._connection, statement, parameters)) + + def _before_commit(self, check: Callable[[], None]) -> None: + if not self._active: + raise ExecutionStoreError("shared_transaction_closed") + self._commit_checks.append(check) + + +def _execute_host_owned( + connection: sqlite3.Connection, statement: str, parameters: Any +) -> sqlite3.Cursor: + """Trusted adapter/helper path; application SQL never receives this operation.""" + connection.set_authorizer(None) + try: + return connection.execute(statement, parameters) + finally: + connection.set_authorizer(_application_authorizer) + def _application_authorizer( operation: int, @@ -153,6 +187,8 @@ def _application_authorizer( trigger: str | None, ) -> int: del database, trigger + if any(name and name.lower().startswith("determa_") for name in (first, second)): + return sqlite3.SQLITE_DENY if operation == sqlite3.SQLITE_PRAGMA and first == "integrity_check" and second is None: return sqlite3.SQLITE_OK return ( @@ -497,11 +533,17 @@ def shared_transaction( self._validate_schema(connection) connection.execute("BEGIN IMMEDIATE") connection.set_authorizer(_application_authorizer) - yield application, _SQLiteTransaction(connection, root_instance_id) + yield ( + application, + _SQLiteTransaction(connection, root_instance_id, application_sql=True), + ) application._close() connection.set_authorizer(None) if not connection.in_transaction: raise ExecutionStoreError(AdapterCode.ADAPTER_CAPABILITY_MISMATCH) + self._validate_schema(connection) + for check in application._commit_checks: + check() connection.commit() except BaseException: application._close() diff --git a/src/determa/state/timers.py b/src/determa/state/timers.py index 0ac920b..fb5e4f2 100644 --- a/src/determa/state/timers.py +++ b/src/determa/state/timers.py @@ -95,6 +95,16 @@ def _time(value: Any, *, duration: bool = False) -> int: return parsed +def _execute( + connection: sqlite3.Connection | SQLiteApplicationTransaction, + statement: str, + parameters: Any = (), +) -> Any: + if isinstance(connection, SQLiteApplicationTransaction): + return connection._execute_host_owned(statement, parameters) + return connection.execute(statement, parameters) + + def _result( request: Mapping[str, Any] | None, record: dict[str, Any] | None, code: str | None = None ) -> dict[str, Any]: @@ -194,12 +204,13 @@ def setup_schema(self) -> None: self.validate_schema() def _schema(self, connection: sqlite3.Connection | SQLiteApplicationTransaction) -> None: - rows = connection.execute( + rows = _execute( + connection, "SELECT type,sql FROM sqlite_master " - "WHERE tbl_name='determa_timer_helpers' ORDER BY type" + "WHERE tbl_name='determa_timer_helpers' ORDER BY type", ).fetchall() - if rows != [("index", None), ("table", _TABLE)] or connection.execute( - "PRAGMA integrity_check" + if rows != [("index", None), ("table", _TABLE)] or _execute( + connection, "PRAGMA integrity_check" ).fetchone() != ("ok",): raise TimerError("timer_capability_mismatch") @@ -210,7 +221,8 @@ def validate_schema(self) -> None: def _load( self, connection: sqlite3.Connection | SQLiteApplicationTransaction ) -> dict[str, Any]: - row = connection.execute( + row = _execute( + connection, "SELECT document FROM determa_timer_helpers WHERE scope_identity=?", (self.scope_identity,), ).fetchone() @@ -260,7 +272,8 @@ def _persist( artifact = seal_timer_records(artifact) if not _validator("timer-record-v1.schema.json").is_valid(artifact): raise TimerError("timer_capability_mismatch") - connection.execute( + _execute( + connection, "INSERT INTO determa_timer_helpers VALUES (?,?) " "ON CONFLICT(scope_identity) DO UPDATE SET document=excluded.document", (self.scope_identity, canonical_bytes(artifact)), @@ -301,6 +314,7 @@ def callback( sql: SQLiteApplicationTransaction, execution: SharedExecutionTransaction ) -> None: record = None + claimed_record = None try: self._schema(sql) artifact = self._load(sql) @@ -330,6 +344,7 @@ def callback( ) if record is None: raise TimerError("timer_not_found") + claimed_record = copy.deepcopy(record) arguments = request["arguments"] if record["attempt_fence"] != arguments["attempt_fence"]: raise TimerError("timer_stale_fence") @@ -347,7 +362,8 @@ def callback( raise TimerError("timer_worker_mismatch") if record["event_id"] != arguments["event_id"]: raise TimerError("timer_event_conflict") - if self._now() >= _time(record["expires_at"]): + claim_expiry = _time(record["expires_at"]) + if self._now() >= claim_expiry: raise TimerError("timer_stale_fence") restored = execution.read_checkpoint() if restored is None: @@ -419,11 +435,22 @@ def callback( } ) self._persist(sql, artifact) + + def check_lease() -> None: + try: + if self._now() >= claim_expiry: + raise TimerError("timer_stale_fence") + except TimerError as error: + raise _TimerReturn(_result(request, claimed_record, error.code)) from error + + sql._before_commit(check_lease) committed.append(result) except TimerError as error: - raise _TimerReturn(_result(request, record, error.code)) from error + raise _TimerReturn(_result(request, claimed_record, error.code)) from error except ExecutionHostError as error: - raise _TimerReturn(_result(request, record, "timer_admission_rejected")) from error + raise _TimerReturn( + _result(request, claimed_record, "timer_admission_rejected") + ) from error try: host.run_shared_transaction(self.root_instance_id, callback) diff --git a/tests/test_sqlite_shared_transaction.py b/tests/test_sqlite_shared_transaction.py index 127d859..9dc4735 100644 --- a/tests/test_sqlite_shared_transaction.py +++ b/tests/test_sqlite_shared_transaction.py @@ -96,3 +96,31 @@ def test_sqlite_default_store_refuses_shared_transaction(tmp_path): host = ExecutionHost(store, _resolver()) with pytest.raises(Exception, match="adapter_capability_mismatch"): host.run_shared_transaction("root", lambda sql, execution: None) + + +@pytest.mark.parametrize( + "statement", + [ + "DROP TRIGGER determa_execution_checkpoints_forbid_delete", + "DELETE FROM determa_execution_checkpoints WHERE root_instance_id='victim'", + "DROP TABLE determa_execution_checkpoints", + "UPDATE determa_execution_checkpoints SET revision='999' WHERE root_instance_id='victim'", + "CREATE TRIGGER app_delete AFTER INSERT ON applications " + "BEGIN DELETE FROM determa_execution_checkpoints; END", + ], +) +def test_application_sql_cannot_change_protected_engine_storage(tmp_path, statement): + store, host = configured(tmp_path) + host.create_v1(load_bundle(MACHINE), "counter", "victim", "create-victim", {}) + before = host.read_checkpoint("victim").document + + def callback(sql, execution): + execution.create_v1(load_bundle(MACHINE), "counter", "create-other", {}) + sql.execute(statement) + sql.execute("INSERT INTO applications VALUES ('trigger-attack')") + + with pytest.raises(sqlite3.DatabaseError, match="(not authorized|prohibited)"): + host.run_shared_transaction("other-root", callback) + assert host.read_checkpoint("victim").document == before + assert host.read_checkpoint("other-root") is None + assert store.health()["healthy"] diff --git a/tests/test_timer_coordinated_admission.py b/tests/test_timer_coordinated_admission.py index 090247d..2d151f3 100644 --- a/tests/test_timer_coordinated_admission.py +++ b/tests/test_timer_coordinated_admission.py @@ -143,7 +143,7 @@ def offline(): assert restarted.execute(complete, principal="worker-A") == result assert restarted.snapshot() == committed assert restarted_host.read_checkpoint("server-1").document == expected - assert calls == ["100", "110", "120"] + assert calls == ["100", "110", "120", "120"] @pytest.mark.parametrize( @@ -197,4 +197,31 @@ def interrupted(boundary): assert host.read_checkpoint("server-1").document == expected # Retained replay aborts the read-only callback before the response fault hook. assert helper.execute(complete, principal="worker-A")["status"] == "accepted" - assert calls == ["100", "110", "120"] + assert calls == ["100", "110", "120", "120"] + + +@pytest.mark.parametrize("fate", ["expired", "offline"]) +def test_lease_is_rechecked_after_staging_and_before_native_commit(tmp_path, fate): + helper, host, complete, expected, now, calls = configured(tmp_path) + before_helper = helper.snapshot() + before_checkpoint = host.read_checkpoint("server-1").document + reads = [] + + def advancing_clock(): + reads.append(now[0]) + if len(reads) == 1: + now[0] = "1000" + return "120" + if fate == "offline": + raise RuntimeError("clock offline") + return now[0] + + helper.trusted_clock = advancing_clock + result = helper.execute(complete, principal="worker-A") + assert result["error_code"] == ( + "timer_stale_fence" if fate == "expired" else "timer_clock_unavailable" + ) + assert result["record_revision"] == "2" and result["attempt_fence"] == "1" + assert helper.snapshot() == before_helper + assert host.read_checkpoint("server-1").document == before_checkpoint + assert reads == ["120", "1000"] From 96da0ff72a17db356be748c5339779ea3206bc48 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 17:01:39 +0000 Subject: [PATCH 12/32] Expose explicit shared SQLite transactions and align timer worker refusals --- .github/workflows/test.yml | 6 +++--- AGENTS.md | 2 +- README.md | 2 +- conformance/pins.py | 2 +- src/determa/state/extensions.py | 1 + src/determa/state/stores/sqlite.py | 5 +++++ src/determa/state/timers.py | 14 +++++++++++--- tests/test_sqlite_shared_transaction.py | 17 +++++++++++++++++ tests/test_timers.py | 4 +--- 9 files changed, 41 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 28d7888..200ca4a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,7 +53,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: c6949440f5b9fd6e06ea5571f4e01681dd59ee04 + ref: dc84ed81ea36a5f2140181a97660477a14347ccc path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -88,7 +88,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: c6949440f5b9fd6e06ea5571f4e01681dd59ee04 + ref: dc84ed81ea36a5f2140181a97660477a14347ccc path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -126,7 +126,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: c6949440f5b9fd6e06ea5571f4e01681dd59ee04 + ref: dc84ed81ea36a5f2140181a97660477a14347ccc path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/AGENTS.md b/AGENTS.md index eba3e97..4b87733 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ The implementation is conformant only when it passes the language-neutral suite. The implementation uses these immutable inputs: - specification: `77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0`; -- conformance: `c6949440f5b9fd6e06ea5571f4e01681dd59ee04` (99 format-1 +- conformance: `dc84ed81ea36a5f2140181a97660477a14347ccc` (99 format-1 core cases, 162 version-1 vectors, 142 durable-host vectors, 49 core inspection vectors, 47 extension negotiation vectors, 7 optional native-provider vectors, 52 runtime-provider vectors, and 487 JSON artifacts). diff --git a/README.md b/README.md index 596bd4e..2bc7f86 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ commit `77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0`. Correctness is determined by 99 format-1 core cases, 162 version-1 vectors, 142 durable-host vectors, 49 core inspection vectors, 47 extension negotiation vectors, 7 native guard-provider inspection vectors, 52 runtime-provider vectors, and 487 manifest artifacts at -conformance commit `c6949440f5b9fd6e06ea5571f4e01681dd59ee04`. The native +conformance commit `dc84ed81ea36a5f2140181a97660477a14347ccc`. The native runtime-provider profile is optional and is exercised through its production adapter. The package metadata is `0.3.0`. Artifact and checkpoint schema version 1 is the diff --git a/conformance/pins.py b/conformance/pins.py index 0f9c49f..fac42a1 100644 --- a/conformance/pins.py +++ b/conformance/pins.py @@ -4,7 +4,7 @@ from pathlib import Path -CONFORMANCE_COMMIT = "c6949440f5b9fd6e06ea5571f4e01681dd59ee04" +CONFORMANCE_COMMIT = "dc84ed81ea36a5f2140181a97660477a14347ccc" SPEC_COMMIT = "77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0" ROOT = Path(__file__).resolve().parent.parent diff --git a/src/determa/state/extensions.py b/src/determa/state/extensions.py index b75f01f..3a484b2 100644 --- a/src/determa/state/extensions.py +++ b/src/determa/state/extensions.py @@ -1302,6 +1302,7 @@ def verify(provider: Any, descriptor: Mapping[str, Any]) -> bool: "file": {"restart_persistent"}, "sqlite": { "durable_single_writer", + "shared_application_transaction", "root_identity_retention", "permanent_receipt_retention", "permanent_outbox_terminal_retention", diff --git a/src/determa/state/stores/sqlite.py b/src/determa/state/stores/sqlite.py index 6d386a0..0b6383d 100644 --- a/src/determa/state/stores/sqlite.py +++ b/src/determa/state/stores/sqlite.py @@ -598,6 +598,7 @@ def sqlite_execution_store_factory(uri: str, configuration: Mapping[str, Any]) - "timeout", "replay_retention", "outbox_retention", + "shared_application_transactions", }: raise ExecutionStoreError(AdapterCode.INVALID_ADAPTER_CONFIGURATION) journal_mode = _single_query(query, "journal_mode", "WAL") @@ -605,6 +606,9 @@ def sqlite_execution_store_factory(uri: str, configuration: Mapping[str, Any]) - timeout_text = _single_query(query, "timeout", "30") replay_retention = _single_query(query, "replay_retention", "bounded") outbox_retention = _single_query(query, "outbox_retention", "none") + shared = _single_query(query, "shared_application_transactions", "false") + if shared not in {"true", "false"}: + raise ExecutionStoreError(AdapterCode.INVALID_ADAPTER_CONFIGURATION) try: timeout = float(timeout_text) except ValueError as exc: @@ -616,4 +620,5 @@ def sqlite_execution_store_factory(uri: str, configuration: Mapping[str, Any]) - timeout=timeout, replay_retention=replay_retention, outbox_retention=outbox_retention, + shared_application_transactions=shared == "true", ) diff --git a/src/determa/state/timers.py b/src/determa/state/timers.py index fb5e4f2..221932e 100644 --- a/src/determa/state/timers.py +++ b/src/determa/state/timers.py @@ -328,6 +328,9 @@ def callback( ), None, ) + claimed_record = copy.deepcopy(record) + if request["arguments"]["worker_principal"] != principal: + raise TimerError("timer_worker_mismatch") previous = next( ( item @@ -488,9 +491,9 @@ def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, An request["root_runtime_id"], ) != (self.scope_identity, self.root_instance_id, self.root_runtime_id): return _result(request, None, "unauthorized_timer_scope") - if request["operation"] in ("claim_fire", "complete_fire") and ( - principal not in self.worker_principals - or request["arguments"]["worker_principal"] != principal + if ( + request["operation"] in ("claim_fire", "complete_fire") + and principal not in self.worker_principals ): return _result(request, None, "unauthorized_timer_scope") if request["operation"] == "complete_fire" and self.coordinated_host is not None: @@ -519,6 +522,11 @@ def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, An ), None, ) + if ( + request["operation"] in ("claim_fire", "complete_fire") + and request["arguments"]["worker_principal"] != principal + ): + return _result(request, record, "timer_worker_mismatch") if previous is not None: return ( copy.deepcopy(previous["result"]) diff --git a/tests/test_sqlite_shared_transaction.py b/tests/test_sqlite_shared_transaction.py index 9dc4735..2c96ec5 100644 --- a/tests/test_sqlite_shared_transaction.py +++ b/tests/test_sqlite_shared_transaction.py @@ -6,6 +6,7 @@ from determa.state import ExecutionHost, ExecutionStoreError, SQLiteExecutionStore, load_bundle from determa.state.stores.base import SHARED_APPLICATION_TRANSACTION +from determa.state.stores.sqlite import sqlite_execution_store_factory from .test_execution_stores import MACHINE, _resolver @@ -98,6 +99,22 @@ def test_sqlite_default_store_refuses_shared_transaction(tmp_path): host.run_shared_transaction("root", lambda sql, execution: None) +def test_bundled_sqlite_uri_requires_an_explicit_shared_transaction_option(tmp_path): + path = tmp_path / "public-uri.sqlite" + ordinary = sqlite_execution_store_factory(f"sqlite://{path}", {}) + ordinary.setup_schema() + selected = sqlite_execution_store_factory( + f"sqlite://{path}?shared_application_transactions=true", {} + ) + assert SHARED_APPLICATION_TRANSACTION in selected.capabilities + assert SHARED_APPLICATION_TRANSACTION not in ordinary.capabilities + for invalid in ("1", "TRUE", "", "yes", "true&shared_application_transactions=false"): + with pytest.raises(ExecutionStoreError, match="invalid_adapter_configuration"): + sqlite_execution_store_factory( + f"sqlite://{path}?shared_application_transactions={invalid}", {} + ) + + @pytest.mark.parametrize( "statement", [ diff --git a/tests/test_timers.py b/tests/test_timers.py index 476cc04..897e96e 100644 --- a/tests/test_timers.py +++ b/tests/test_timers.py @@ -244,9 +244,7 @@ def test_expired_claim_without_commit_fate_proof_never_grants_a_new_fence(tmp_pa refused = worker_helper(path, lambda: now[0]).execute(retry, principal="worker-B") assert refused["error_code"] == "delivery_ambiguous" and refused["attempt_fence"] == "1" assert installed.snapshot() == before - assert ( - installed.execute(retry, principal="worker-A")["error_code"] == "unauthorized_timer_scope" - ) + assert installed.execute(retry, principal="worker-A")["error_code"] == "timer_worker_mismatch" assert installed.snapshot() == before From 0d095bb1cf175d874e3310fbc16945d9efe4c325 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 17:04:01 +0000 Subject: [PATCH 13/32] Prove coordinated timer restart after actual SIGKILL at native commit cuts --- tests/test_timer_coordinated_admission.py | 110 ++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/tests/test_timer_coordinated_admission.py b/tests/test_timer_coordinated_admission.py index 2d151f3..055e5ad 100644 --- a/tests/test_timer_coordinated_admission.py +++ b/tests/test_timer_coordinated_admission.py @@ -1,6 +1,9 @@ from __future__ import annotations import copy +import multiprocessing +import os +import signal import pytest @@ -13,6 +16,113 @@ from .test_timers import bundle, request +def _killed_timer_worker(path, complete, boundary, control): + definition = bundle() + host = ExecutionHost( + SQLiteExecutionStore( + path, replay_retention="permanent", shared_application_transactions=True + ), + MemoryArtifactResolver(definitions={definition.fingerprint: definition}), + ) + helper = SQLiteTimerHelper( + path, + definition, + scope_identity=complete["scope_identity"], + root_instance_id=complete["root_instance_id"], + root_runtime_id=complete["root_runtime_id"], + principals=frozenset({"operator", "worker-A"}), + worker_principals=frozenset({"worker-A"}), + trusted_clock=lambda: "120", + coordinated_host=host, + ) + + def pause(): + control.send(boundary) + control.recv() + raise AssertionError("the parent must kill this process at the selected boundary") + + if boundary == "both_writes_staged": + persist = helper._persist + + def pause_after_staging(sql, artifact): + persist(sql, artifact) + pause() + + helper._persist = pause_after_staging + else: + + def pause_after_commit(observed): + if observed == "after_commit_before_response": + pause() + + host.fault_injector = pause_after_commit + helper.execute(complete, principal="worker-A") + + +@pytest.mark.skipif(os.name != "posix", reason="requires actual SIGKILL process termination") +@pytest.mark.parametrize("boundary", ["both_writes_staged", "after_commit_before_response"]) +def test_sigkill_restart_resolves_actual_native_timer_and_admission_fate(tmp_path, boundary): + helper, host, complete, expected, now, calls = configured(tmp_path) + before_helper = helper.snapshot() + before_checkpoint = host.read_checkpoint("server-1").document + context = multiprocessing.get_context("spawn") + parent, child = context.Pipe() + worker = context.Process( + target=_killed_timer_worker, args=(helper.path, complete, boundary, child) + ) + worker.start() + child.close() + try: + assert parent.poll(30), "worker never reached the selected native boundary" + assert parent.recv() == boundary + worker.kill() + worker.join(10) + assert worker.exitcode == -signal.SIGKILL + finally: + if worker.is_alive(): + worker.kill() + worker.join(10) + parent.close() + worker.close() + + # Reopen all production objects: no child response or exception is recovery proof. + restarted_host = ExecutionHost( + SQLiteExecutionStore( + helper.path, replay_retention="permanent", shared_application_transactions=True + ), + host.artifact_resolver, + ) + restarted = SQLiteTimerHelper( + helper.path, + helper.bundle, + scope_identity=helper.scope_identity, + root_instance_id=helper.root_instance_id, + root_runtime_id=helper.root_runtime_id, + principals=helper.principals, + worker_principals=helper.worker_principals, + trusted_clock=lambda: "120", + coordinated_host=restarted_host, + ) + if boundary == "both_writes_staged": + assert restarted.snapshot() == before_helper + assert restarted_host.read_checkpoint("server-1").document == before_checkpoint + else: + assert restarted.snapshot()["records"][0]["state"] == "fired" + assert restarted_host.read_checkpoint("server-1").document == expected + + def offline(): + raise AssertionError("committed replay must not read the clock") + + restarted.trusted_clock = offline + result = restarted.execute(complete, principal="worker-A") + assert result["status"] == "accepted" and result["delivery_state"] == "admitted" + committed = restarted.snapshot() + assert restarted_host.read_checkpoint("server-1").document == expected + assert restarted.execute(complete, principal="worker-A") == result + assert restarted.snapshot() == committed + assert restarted_host.read_checkpoint("server-1").document == expected + + def configured(tmp_path): path = tmp_path / "coordinated.sqlite" definition = bundle() From 5d284e83bc1f120a5d427198ce1af9c052542f83 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 17:15:29 +0000 Subject: [PATCH 14/32] Retain storage-bound timer origin and immutable native operation evidence --- src/determa/state/timers.py | 214 ++++++++++++++++++++++++++++++++---- tests/test_timers.py | 80 +++++++++++++- 2 files changed, 272 insertions(+), 22 deletions(-) diff --git a/src/determa/state/timers.py b/src/determa/state/timers.py index 221932e..d3bdf14 100644 --- a/src/determa/state/timers.py +++ b/src/determa/state/timers.py @@ -33,6 +33,29 @@ "CREATE TABLE determa_timer_helpers " "(scope_identity TEXT PRIMARY KEY NOT NULL, document BLOB NOT NULL)" ) +_ORIGIN_TABLE = ( + "CREATE TABLE determa_timer_origin " + "(singleton INTEGER PRIMARY KEY NOT NULL CHECK(singleton=1), configuration BLOB NOT NULL)" +) +_COMMIT_TABLE = ( + "CREATE TABLE determa_timer_commits " + "(scope_identity TEXT NOT NULL, sequence INTEGER NOT NULL, operation_id TEXT NOT NULL, " + "journal BLOB NOT NULL, PRIMARY KEY(scope_identity, sequence), " + "UNIQUE(scope_identity, operation_id))" +) +_NATIVE_TABLES = { + "determa_timer_helpers": _TABLE, + "determa_timer_origin": _ORIGIN_TABLE, + "determa_timer_commits": _COMMIT_TABLE, +} +_NATIVE_TRIGGERS = { + f"{table}_forbid_{operation.lower()}": ( + f"CREATE TRIGGER {table}_forbid_{operation.lower()} BEFORE {operation} ON {table} " + "BEGIN SELECT RAISE(ABORT, 'timer_native_evidence_immutable'); END" + ) + for table in ("determa_timer_origin", "determa_timer_commits") + for operation in ("UPDATE", "DELETE") +} _IDENTITIES = ( "operation", "operation_id", @@ -200,23 +223,78 @@ def _connect(self) -> sqlite3.Connection: def setup_schema(self) -> None: with closing(self._connect()) as connection: - connection.execute(_TABLE.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1)) - self.validate_schema() + connection.execute("BEGIN IMMEDIATE") + try: + existing = connection.execute( + "SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'determa_timer_%'" + ).fetchone()[0] + if existing: + # Never reconstruct origin or retained evidence around prior records. + self._schema(connection) + else: + for statement in (*_NATIVE_TABLES.values(), *_NATIVE_TRIGGERS.values()): + connection.execute(statement) + connection.execute( + "INSERT INTO determa_timer_origin VALUES (1,?)", + (canonical_bytes(self._configuration()),), + ) + self._schema(connection) + connection.commit() + finally: + connection.rollback() + + def _configuration(self) -> dict[str, Any]: + return { + "native_timer_configuration_version": 1, + "storage_path": self.path, + "scope_identity": self.scope_identity, + "root_instance_id": self.root_instance_id, + "root_runtime_id": self.root_runtime_id, + "validated_bundle_fingerprint": self.bundle.fingerprint, + "principals": sorted(self.principals), + "worker_principals": sorted(self.worker_principals), + "claim_lease_nanoseconds": str(self.claim_lease_nanoseconds), + "admission_strategy": "coordinated_sqlite" + if self.coordinated_host is not None + else "records_only", + } def _schema(self, connection: sqlite3.Connection | SQLiteApplicationTransaction) -> None: rows = _execute( connection, - "SELECT type,sql FROM sqlite_master " - "WHERE tbl_name='determa_timer_helpers' ORDER BY type", + "SELECT name,type,sql FROM sqlite_master WHERE tbl_name IN " + "('determa_timer_helpers','determa_timer_origin','determa_timer_commits') " + "OR name LIKE 'determa_timer_%' ORDER BY name", ).fetchall() - if rows != [("index", None), ("table", _TABLE)] or _execute( - connection, "PRAGMA integrity_check" - ).fetchone() != ("ok",): + actual = {name: (kind, sql) for name, kind, sql in rows if sql is not None} + expected = { + **{name: ("table", sql) for name, sql in _NATIVE_TABLES.items()}, + **{name: ("trigger", sql) for name, sql in _NATIVE_TRIGGERS.items()}, + } + if ( + actual != expected + or len(rows) != 10 + or _execute(connection, "PRAGMA integrity_check").fetchone() != ("ok",) + ): raise TimerError("timer_capability_mismatch") + origin = _execute(connection, "SELECT configuration FROM determa_timer_origin").fetchall() + if origin != [(canonical_bytes(self._configuration()),)]: + raise TimerError("timer_capability_mismatch") + for table in ("determa_timer_helpers", "determa_timer_commits"): + if _execute( + connection, + f"SELECT COUNT(*) FROM {table} WHERE scope_identity!=?", + (self.scope_identity,), + ).fetchone() != (0,): + raise TimerError("timer_capability_mismatch") def validate_schema(self) -> None: with closing(self._connect()) as connection: - self._schema(connection) + connection.execute("BEGIN") + try: + self._schema(connection) + finally: + connection.rollback() def _load( self, connection: sqlite3.Connection | SQLiteApplicationTransaction @@ -227,7 +305,7 @@ def _load( (self.scope_identity,), ).fetchone() if row is None: - return seal_timer_records( + artifact = seal_timer_records( { "timer_artifact_format": "determa.timer_records", "timer_artifact_schema_version": 1, @@ -235,13 +313,14 @@ def _load( "operation_receipts": [], } ) - try: - parsed, raw = strict_json(bytes(row[0])) - if not isinstance(parsed, dict) or raw != canonical_bytes(parsed): - raise TimerError("timer_capability_mismatch") - artifact: dict[str, Any] = parsed - except (ArtifactError, TypeError, ValueError) as error: - raise TimerError("timer_capability_mismatch") from error + else: + try: + parsed, raw = strict_json(bytes(row[0])) + if not isinstance(parsed, dict) or raw != canonical_bytes(parsed): + raise TimerError("timer_capability_mismatch") + artifact = parsed + except (ArtifactError, TypeError, ValueError) as error: + raise TimerError("timer_capability_mismatch") from error if ( not _validator("timer-record-v1.schema.json").is_valid(artifact) or seal_timer_records(artifact) != artifact @@ -249,12 +328,105 @@ def _load( raise TimerError("timer_capability_mismatch") if any(record["scope_identity"] != self.scope_identity for record in artifact["records"]): raise TimerError("timer_capability_mismatch") + self._history(connection, artifact) return artifact + def _history( + self, + connection: sqlite3.Connection | SQLiteApplicationTransaction, + artifact: dict[str, Any], + ) -> None: + rows = _execute( + connection, + "SELECT sequence,operation_id,journal FROM determa_timer_commits " + "WHERE scope_identity=? ORDER BY sequence", + (self.scope_identity,), + ).fetchall() + if len(rows) != len(artifact["operation_receipts"]): + raise TimerError("timer_capability_mismatch") + if not rows and artifact["records"]: + raise TimerError("timer_capability_mismatch") + last_digest = None + for sequence, (native_sequence, operation_id, raw) in enumerate(rows, start=1): + try: + journal, original = strict_json(bytes(raw)) + if ( + native_sequence != sequence + or not isinstance(journal, dict) + or original != canonical_bytes(journal) + or set(journal) != {"request", "result", "timer_artifact_digest"} + ): + raise TimerError("timer_capability_mismatch") + request = journal["request"] + result = journal["result"] + receipt = artifact["operation_receipts"][sequence - 1] + if ( + not _validator("timer-helper-operation-v1.schema.json", "request").is_valid( + request + ) + or request["request_digest"] != timer_request_digest(request) + or request["operation_id"] != operation_id + or request["scope_identity"] != self.scope_identity + or request["root_instance_id"] != self.root_instance_id + or request["root_runtime_id"] != self.root_runtime_id + or result["status"] != "accepted" + or any(result[key] != request[key] for key in _IDENTITIES) + or result["result_digest"] + != hash_value( + [ + "determa-timer-result-1", + request["request_digest"], + {key: value for key, value in result.items() if key != "result_digest"}, + ] + ) + or receipt + != { + "operation_id": operation_id, + "request_digest": request["request_digest"], + "result": result, + } + ): + raise TimerError("timer_capability_mismatch") + last_digest = journal["timer_artifact_digest"] + except (ArtifactError, TypeError, ValueError, KeyError) as error: + raise TimerError("timer_capability_mismatch") from error + if last_digest is not None and last_digest != artifact["timer_artifact_digest"]: + raise TimerError("timer_capability_mismatch") + + def _record_commit( + self, + connection: sqlite3.Connection | SQLiteApplicationTransaction, + request: Mapping[str, Any], + result: dict[str, Any], + artifact: dict[str, Any], + ) -> None: + self._persist(connection, artifact) + sealed = seal_timer_records(artifact) + _execute( + connection, + "INSERT INTO determa_timer_commits VALUES (?,?,?,?)", + ( + self.scope_identity, + len(artifact["operation_receipts"]), + request["operation_id"], + canonical_bytes( + { + "request": request, + "result": result, + "timer_artifact_digest": sealed["timer_artifact_digest"], + } + ), + ), + ) + def snapshot(self) -> dict[str, Any]: with closing(self._connect()) as connection: - self._schema(connection) - return self._load(connection) + connection.execute("BEGIN") + try: + self._schema(connection) + return self._load(connection) + finally: + connection.rollback() def _persist( self, @@ -437,7 +609,7 @@ def callback( "result": result, } ) - self._persist(sql, artifact) + self._record_commit(sql, request, result, artifact) def check_lease() -> None: try: @@ -501,8 +673,8 @@ def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, An record = None connection = self._connect() try: - self._schema(connection) connection.execute("BEGIN IMMEDIATE") + self._schema(connection) artifact = self._load(connection) record = next( ( @@ -635,7 +807,7 @@ def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, An "result": result, } ) - self._persist(connection, artifact) + self._record_commit(connection, request, result, artifact) connection.commit() return result except TimerError as error: diff --git a/tests/test_timers.py b/tests/test_timers.py index 897e96e..e2325a8 100644 --- a/tests/test_timers.py +++ b/tests/test_timers.py @@ -7,11 +7,89 @@ import pytest from determa.state import load_bundle -from determa.state.timers import SQLiteTimerHelper, timer_request_digest +from determa.state.timers import ( + SQLiteTimerHelper, + TimerError, + seal_timer_records, + timer_request_digest, +) _ROOT_RUNTIME = "sha256:9b0e3238e782cb7d912febc1acc4e4c03a121d201764d0bc9658b969feac4dc8" +@pytest.mark.parametrize("damage", ["record", "receipt", "unretained_record"]) +def test_resealed_portable_records_do_not_replace_retained_native_evidence(tmp_path, damage): + import sqlite3 + + from determa.state.wire import canonical_bytes + + installed = helper(tmp_path / "native-evidence.sqlite", lambda: "100") + installed.setup_schema() + assert installed.execute(request(), principal="operator")["status"] == "accepted" + artifact = installed.snapshot() + if damage == "record": + artifact["records"][0]["deadline_at"] = "999" + elif damage == "receipt": + artifact["operation_receipts"][0]["result"]["record_revision"] = "999" + else: + artifact["operation_receipts"] = [] + with sqlite3.connect(installed.path) as connection: + connection.execute( + "UPDATE determa_timer_helpers SET document=?", + (canonical_bytes(seal_timer_records(artifact)),), + ) + result = installed.execute(request("read_timer", "read-A", {}), principal="operator") + assert result["error_code"] == "timer_capability_mismatch" + + +def test_native_origin_is_storage_bound_and_setup_cannot_repair_missing_evidence(tmp_path): + import sqlite3 + + original = helper(tmp_path / "original.sqlite", lambda: "100") + original.setup_schema() + original.execute(request(), principal="operator") + before = original.snapshot() + copied_path = tmp_path / "copied.sqlite" + with sqlite3.connect(original.path) as source, sqlite3.connect(copied_path) as copied: + source.backup(copied) + inert = helper(copied_path, lambda: "100") + with pytest.raises(TimerError, match="timer_capability_mismatch"): + inert.snapshot() + with sqlite3.connect(copied_path) as connection: + connection.execute("DROP TABLE determa_timer_origin") + recorded = connection.execute("SELECT document FROM determa_timer_helpers").fetchall() + with pytest.raises(TimerError, match="timer_capability_mismatch"): + inert.setup_schema() + with sqlite3.connect(copied_path) as connection: + assert connection.execute( + "SELECT COUNT(*) FROM sqlite_master WHERE name='determa_timer_origin'" + ).fetchone() == (0,) + assert ( + connection.execute("SELECT document FROM determa_timer_helpers").fetchall() == recorded + ) + assert original.snapshot() == before + + +def test_native_operation_history_is_immutable_and_configuration_changes_fail_closed(tmp_path): + import sqlite3 + + installed = helper(tmp_path / "configuration.sqlite", lambda: "100") + installed.setup_schema() + installed.execute(request(), principal="operator") + with sqlite3.connect(installed.path) as connection: + for statement in ( + "DELETE FROM determa_timer_commits", + "UPDATE determa_timer_commits SET operation_id='changed'", + "DELETE FROM determa_timer_origin", + "UPDATE determa_timer_origin SET configuration=X'00'", + ): + with pytest.raises(sqlite3.IntegrityError, match="timer_native_evidence_immutable"): + connection.execute(statement) + installed.claim_lease_nanoseconds += 1 + result = installed.execute(request("read_timer", "read-A", {}), principal="operator") + assert result["error_code"] == "timer_capability_mismatch" + + def bundle(): return load_bundle( { From c7a17b8c405ba36ddacf8ff41ec9e6acf2396124 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 18:02:35 +0000 Subject: [PATCH 15/32] Implement strict local timer reclaim from native transaction fate --- src/determa/state/timers.py | 169 +++++++++++++- tests/test_timer_coordinated_admission.py | 4 +- tests/test_timer_native_fate.py | 261 ++++++++++++++++++++++ 3 files changed, 422 insertions(+), 12 deletions(-) create mode 100644 tests/test_timer_native_fate.py diff --git a/src/determa/state/timers.py b/src/determa/state/timers.py index d3bdf14..1404deb 100644 --- a/src/determa/state/timers.py +++ b/src/determa/state/timers.py @@ -1,13 +1,15 @@ """Explicit foreground SQLite timer records, separate from portable checkpoints. This implementation checkpoint supports durable schedule/cancel/read, initial claims -and coordinated SQLite admission. Proved-fate reclaim, verified provider installation, +and coordinated SQLite admission with strictly local native-fate reclaim. Verified +provider installation, archive and recovery integration remain unfinished; no completed profile is advertised. """ from __future__ import annotations import copy +import hashlib import json import re import sqlite3 @@ -16,6 +18,7 @@ from pathlib import Path from typing import Any +from .checkpoint_v1 import restore_execution_checkpoint_v1 from .definition import Bundle from .engine import _normalize_payload from .errors import ArtifactError @@ -25,6 +28,7 @@ SharedExecutionTransaction, delivery_request_digest, ) +from .stores.base import ExecutionStoreError from .stores.sqlite import SQLiteApplicationTransaction, SQLiteExecutionStore from .wire import _schema_registry, canonical_bytes, decoded_typed_value, hash_value, strict_json @@ -243,9 +247,33 @@ def setup_schema(self) -> None: finally: connection.rollback() + def _implementation_digest(self) -> str: + # An existing native origin cannot silently change completion implementation. + # Independent loaded-source registration is still required before a profile claim. + directory = Path(__file__).parent + closure = b"".join( + path.relative_to(directory).as_posix().encode() + b"\0" + path.read_bytes() + for path in sorted(directory.rglob("*")) + if path.is_file() and path.suffix in {".py", ".json"} + ) + return "sha256:" + hashlib.sha256(closure).hexdigest() + def _configuration(self) -> dict[str, Any]: return { "native_timer_configuration_version": 1, + "implementation_digest": self._implementation_digest(), + "coordinated_store_configuration": { + "journal_mode": self.coordinated_host.store.journal_mode, + "synchronous": self.coordinated_host.store.synchronous, + "replay_retention": self.coordinated_host.store.replay_retention, + "outbox_retention": self.coordinated_host.store.outbox_retention, + "shared_application_transactions": ( + self.coordinated_host.store.shared_application_transactions + ), + } + if self.coordinated_host is not None + and type(self.coordinated_host.store) is SQLiteExecutionStore + else None, "storage_path": self.path, "scope_identity": self.scope_identity, "root_instance_id": self.root_instance_id, @@ -471,6 +499,96 @@ def _now(self) -> int: except Exception as error: raise TimerError("timer_clock_unavailable") from error + def _prove_local_uncommitted_fire( + self, connection: sqlite3.Connection, artifact: dict[str, Any], record: dict[str, Any] + ) -> None: + """Resolve the previous native transaction before committing a replacement fence. + + Caller time, absence of an event, and portable claims are not this proof. + The caller owns BEGIN IMMEDIATE, and _load has validated the complete immutable + native commit history against this storage-bound origin. Every completion by + this exact coordinated implementation commits admission and fired evidence + together. The preceding writer has therefore resolved before this decision. + """ + host = self.coordinated_host + if ( + type(self) is not SQLiteTimerHelper + or type(host) is not ExecutionHost + or type(host.store) is not SQLiteExecutionStore + or Path(host.store.path).resolve() != Path(self.path) + or not host.store.shared_application_transactions + or host.store.journal_mode != "WAL" + or host.store.synchronous != "FULL" + or host.store.replay_retention != "permanent" + or not connection.in_transaction + or any( + getattr(SharedExecutionTransaction, name) is not method + or getattr(method, "__code__", None) is not code + for name, method, code in _FATE_SHARED_METHODS + ) + or any( + getattr(getattr(instance, name), "__func__", None) is not method + or getattr(method, "__code__", None) is not code + for instance, methods in ( + (host, _FATE_HOST_METHODS), + (host.store, _FATE_STORE_METHODS), + ) + for name, method, code in methods + ) + or any( + getattr(getattr(self, name), "__func__", None) is not method + or getattr(method, "__code__", None) is not code + for name, method, code in _FATE_METHODS + ) + ): + raise TimerError("delivery_ambiguous") + retained_claim = any( + receipt["result"]["operation"] == "claim_fire" + and receipt["result"]["timer_id"] == record["timer_id"] + and receipt["result"]["record_revision"] == record["revision"] + and receipt["result"]["attempt_fence"] == record["attempt_fence"] + and receipt["result"]["expires_at"] == record["expires_at"] + for receipt in artifact["operation_receipts"] + ) + if not retained_claim or any( + receipt["result"]["operation"] == "complete_fire" + and receipt["result"]["timer_id"] == record["timer_id"] + for receipt in artifact["operation_receipts"] + ): + raise TimerError("delivery_ambiguous") + try: + host.store._validate_schema(connection) + row = connection.execute( + "SELECT revision,checkpoint_digest,checkpoint FROM determa_execution_checkpoints " + "WHERE root_instance_id=?", + (self.root_instance_id,), + ).fetchone() + if row is None: + raise TimerError("delivery_ambiguous") + checkpoint = restore_execution_checkpoint_v1( + bytes(row[2]), host.artifact_resolver + ).document + aggregate = checkpoint["root_record"].get("aggregate_state") + if ( + checkpoint["revision"] != row[0] + or checkpoint["execution_checkpoint_digest"] != row[1] + or checkpoint["root_instance_id"] != self.root_instance_id + or not aggregate + or aggregate["root_runtime_id"] != self.root_runtime_id + or aggregate["validated_bundle_fingerprint"] != self.bundle.fingerprint + or checkpoint["replay_retention"]["mode"] != "permanent" + or checkpoint["replay_retention"]["pruned_through_receipt_sequence"] is not None + # An independently admitted matching event is not helper completion; + # never promote it or infer safe recovery from its receipt or absence. + or any( + receipt.get("event_id") == record["event_id"] + for receipt in checkpoint["operation_receipts"] + ) + ): + raise TimerError("delivery_ambiguous") + except (ArtifactError, ExecutionStoreError, sqlite3.Error) as error: + raise TimerError("delivery_ambiguous") from error + def _complete(self, request: Mapping[str, Any], principal: str) -> dict[str, Any]: host = self.coordinated_host assert host is not None @@ -776,15 +894,9 @@ def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, An return _result(request, record, "timer_cancelled") now = self._now() if record["state"] == "claimed": - # A timeout is not proof that a previous admission did not commit. - # Reclaim is refused until coordinated fate verification is implemented. - return _result( - request, - record, - "delivery_ambiguous" - if now >= _time(record["expires_at"]) - else "timer_fire_in_progress", - ) + if now < _time(record["expires_at"]): + return _result(request, record, "timer_fire_in_progress") + self._prove_local_uncommitted_fire(connection, artifact, record) if now < _time(record["deadline_at"]): return _result(request, record, "timer_not_due") expires_at = now + self.claim_lease_nanoseconds @@ -815,3 +927,40 @@ def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, An finally: connection.rollback() connection.close() + + +# Trusted local factory methods remain bound while an instance proves native fate. +# This guard supplements native origin/history; it is not external registration proof. +_FATE_METHODS = tuple( + (name, getattr(SQLiteTimerHelper, name), getattr(SQLiteTimerHelper, name).__code__) + for name in ( + "execute", + "_complete", + "_load", + "_history", + "_record_commit", + "_persist", + "_schema", + "_configuration", + "_implementation_digest", + "_prove_local_uncommitted_fire", + ) +) + +_FATE_HOST_METHODS = tuple( + (name, getattr(ExecutionHost, name), getattr(ExecutionHost, name).__code__) + for name in ("run_shared_transaction", "_bound", "_restore", "_stage_replace", "admit_v1") +) +_FATE_STORE_METHODS = tuple( + (name, getattr(SQLiteExecutionStore, name), getattr(SQLiteExecutionStore, name).__code__) + for name in ("shared_transaction", "_connect", "_validate_schema") +) + +_FATE_SHARED_METHODS = tuple( + ( + name, + getattr(SharedExecutionTransaction, name), + getattr(SharedExecutionTransaction, name).__code__, + ) + for name in ("admit_v1", "read_checkpoint", "_stage", "_finish") +) diff --git a/tests/test_timer_coordinated_admission.py b/tests/test_timer_coordinated_admission.py index 055e5ad..e601760 100644 --- a/tests/test_timer_coordinated_admission.py +++ b/tests/test_timer_coordinated_admission.py @@ -123,12 +123,12 @@ def offline(): assert restarted_host.read_checkpoint("server-1").document == expected -def configured(tmp_path): +def configured(tmp_path, replay_retention="permanent"): path = tmp_path / "coordinated.sqlite" definition = bundle() resolver = MemoryArtifactResolver(definitions={definition.fingerprint: definition}) store = SQLiteExecutionStore( - path, replay_retention="permanent", shared_application_transactions=True + path, replay_retention=replay_retention, shared_application_transactions=True ) store.setup_schema() host = ExecutionHost(store, resolver) diff --git a/tests/test_timer_native_fate.py b/tests/test_timer_native_fate.py new file mode 100644 index 0000000..3dd6215 --- /dev/null +++ b/tests/test_timer_native_fate.py @@ -0,0 +1,261 @@ +"""Recovery uses native atomic history under its writer lock, never caller fate.""" + +from __future__ import annotations + +import copy +import multiprocessing +import os +import signal +import threading +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from determa.state import ExecutionHost, SQLiteExecutionStore +from determa.state.host import delivery_request_digest +from determa.state.timers import SQLiteTimerHelper, timer_request_digest + +from .test_timer_coordinated_admission import _killed_timer_worker, configured + + +def reclaim(complete, revision="2"): + value = copy.deepcopy(complete) + value.update(operation="claim_fire", operation_id="reclaim-B") + value["arguments"] = { + "expected_revision": revision, + "clock_basis": "unix_nanoseconds", + "worker_principal": "worker-A", + } + value["request_digest"] = timer_request_digest(value) + return value + + +def reopened(helper, host, clock): + return SQLiteTimerHelper( + helper.path, + helper.bundle, + scope_identity=helper.scope_identity, + root_instance_id=helper.root_instance_id, + root_runtime_id=helper.root_runtime_id, + principals=helper.principals, + worker_principals=helper.worker_principals, + trusted_clock=clock, + coordinated_host=host, + ) + + +def test_expired_local_claim_reclaims_with_new_fence_then_rejects_old_worker(tmp_path): + helper, host, complete, expected, now, _ = configured(tmp_path) + before_checkpoint = host.read_checkpoint("server-1").document + now[0] = "130" + result = helper.execute(reclaim(complete), principal="worker-A") + assert result["status"] == "accepted" and result["attempt_fence"] == "2" + assert result["record_revision"] == "3" and result["expires_at"] == "150" + assert host.read_checkpoint("server-1").document == before_checkpoint + before = helper.snapshot() + assert helper.execute(complete, principal="worker-A")["error_code"] == "timer_stale_fence" + assert helper.snapshot() == before + now[0] = "140" + replacement = copy.deepcopy(complete) + replacement["operation_id"] = "complete-B" + replacement["arguments"].update(expected_revision="3", attempt_fence="2") + replacement["request_digest"] = timer_request_digest(replacement) + assert helper.execute(replacement, principal="worker-A")["status"] == "accepted" + assert host.read_checkpoint("server-1").document == expected + # Original reclaim's exact receipt remains replayable after fired state and offline clock. + helper.trusted_clock = lambda: (_ for _ in ()).throw(AssertionError("replay read clock")) + assert helper.execute(reclaim(complete), principal="worker-A") == result + + +@pytest.mark.parametrize("changed_content", [False, True]) +def test_independently_admitted_matching_id_never_becomes_helper_fate_proof( + tmp_path, changed_content +): + helper, host, complete, _, now, _ = configured(tmp_path) + before = host.read_checkpoint("server-1").document + envelope = { + "event": "received", + "event_id": complete["arguments"]["event_id"], + "cause_id": complete["arguments"]["event_id"], + "source": {"host": True}, + "target": { + "root": {"root_instance_id": "server-1", "root_runtime_id": helper.root_runtime_id} + }, + "payload": ["map", []], + } + if changed_content: + envelope["correlation_id"] = "unrelated" + host.admit_v1( + "server-1", + [ + { + "delivery_mode": "input", + "envelope": envelope, + "envelope_digest": delivery_request_digest("server-1", "input", envelope), + } + ], + expected_revision=before["revision"], + expected_checkpoint_digest=before["execution_checkpoint_digest"], + ) + before_helper, before_root = helper.snapshot(), host.read_checkpoint("server-1").document + now[0] = "130" + assert ( + helper.execute(reclaim(complete), principal="worker-A")["error_code"] + == "delivery_ambiguous" + ) + assert helper.snapshot() == before_helper + assert host.read_checkpoint("server-1").document == before_root + + +@pytest.mark.parametrize("changed", ["implementation", "store_policy", "clock"]) +def test_changed_implementation_policy_or_invalid_clock_refuses_without_reclaim(tmp_path, changed): + helper, host, complete, _, now, _ = configured(tmp_path) + before = helper.snapshot() + now[0] = "130" + if changed == "implementation": + host.run_shared_transaction = lambda *_: None + code = "delivery_ambiguous" + elif changed == "store_policy": + host.store.replay_retention = "bounded" + code = "timer_capability_mismatch" + else: + now[0] = "invalid" + code = "timer_clock_unavailable" + assert helper.execute(reclaim(complete), principal="worker-A")["error_code"] == code + if changed == "store_policy": + host.store.replay_retention = "permanent" + assert helper.snapshot() == before + + +def test_completion_staged_before_reclaim_rolls_back_at_expiry_then_new_fence_wins(tmp_path): + helper, host, complete, _, now, _ = configured(tmp_path) + contender = reopened(helper, host, lambda: now[0]) + staged, release = threading.Event(), threading.Event() + persist = helper._persist + + def stop_after_staging(sql, artifact): + persist(sql, artifact) + staged.set() + assert release.wait(20), "test did not release staged transaction" + + helper._persist = stop_after_staging + with ThreadPoolExecutor(max_workers=2) as pool: + previous = pool.submit(helper.execute, complete, principal="worker-A") + try: + assert staged.wait(20), "completion did not stage actual native writes" + now[0] = "130" + replacement = pool.submit(contender.execute, reclaim(complete), principal="worker-A") + finally: + release.set() + assert previous.result(20)["error_code"] == "timer_stale_fence" + assert replacement.result(20)["attempt_fence"] == "2" + assert contender.snapshot()["records"][0]["state"] == "claimed" + assert not any( + item.get("event_id") == complete["arguments"]["event_id"] + for item in host.read_checkpoint("server-1").document["operation_receipts"] + ) + + +@pytest.mark.skipif(os.name != "posix", reason="requires actual SIGKILL") +@pytest.mark.parametrize("boundary", ["both_writes_staged", "after_commit_before_response"]) +def test_sigkill_native_fate_reclaim_or_terminal_replay(tmp_path, boundary): + helper, host, complete, expected, now, _ = configured(tmp_path) + context = multiprocessing.get_context("spawn") + parent, child = context.Pipe() + worker = context.Process( + target=_killed_timer_worker, args=(helper.path, complete, boundary, child) + ) + worker.start() + child.close() + try: + assert parent.poll(30), "worker did not reach native boundary" + assert parent.recv() == boundary + worker.kill() + worker.join(10) + assert worker.exitcode == -signal.SIGKILL + finally: + if worker.is_alive(): + worker.kill() + worker.join(10) + parent.close() + worker.close() + restarted_host = ExecutionHost( + SQLiteExecutionStore( + helper.path, replay_retention="permanent", shared_application_transactions=True + ), + host.artifact_resolver, + ) + now[0] = "130" + restarted = reopened(helper, restarted_host, lambda: now[0]) + if boundary == "both_writes_staged": + result = restarted.execute(reclaim(complete), principal="worker-A") + assert result["status"] == "accepted" and result["attempt_fence"] == "2" + before = restarted.snapshot() + assert ( + restarted.execute(complete, principal="worker-A")["error_code"] == "timer_stale_fence" + ) + assert restarted.snapshot() == before + replacement = copy.deepcopy(complete) + replacement["operation_id"] = "complete-B" + replacement["arguments"].update(expected_revision="3", attempt_fence="2") + replacement["request_digest"] = timer_request_digest(replacement) + now[0] = "140" + assert restarted.execute(replacement, principal="worker-A")["status"] == "accepted" + else: + before = restarted.snapshot() + assert ( + restarted.execute(reclaim(complete, "3"), principal="worker-A")["error_code"] + == "timer_already_fired" + ) + assert restarted.snapshot() == before + assert restarted.execute(complete, principal="worker-A")["status"] == "accepted" + assert restarted_host.read_checkpoint("server-1").document == expected + assert restarted.snapshot()["records"][0]["state"] == "fired" + + +def test_bounded_receipt_history_cannot_prove_uncommitted_fire(tmp_path): + helper, host, complete, _, now, _ = configured(tmp_path, replay_retention="bounded") + before_helper, before_root = helper.snapshot(), host.read_checkpoint("server-1").document + now[0] = "130" + assert ( + helper.execute(reclaim(complete), principal="worker-A")["error_code"] + == "delivery_ambiguous" + ) + assert helper.snapshot() == before_helper + assert host.read_checkpoint("server-1").document == before_root + + +def test_changed_shared_admission_implementation_cannot_establish_native_fate( + tmp_path, monkeypatch +): + from determa.state.host import SharedExecutionTransaction + + helper, _, complete, _, now, _ = configured(tmp_path) + before = helper.snapshot() + now[0] = "130" + monkeypatch.setattr(SharedExecutionTransaction, "admit_v1", lambda *_: None) + assert ( + helper.execute(reclaim(complete), principal="worker-A")["error_code"] + == "delivery_ambiguous" + ) + assert helper.snapshot() == before + + +@pytest.mark.parametrize("missing", ["determa_timer_origin", "determa_timer_commits"]) +def test_missing_origin_or_native_history_cannot_be_rebuilt_for_reclaim(tmp_path, missing): + import sqlite3 + + helper, _, complete, _, now, _ = configured(tmp_path) + now[0] = "130" + with sqlite3.connect(helper.path) as connection: + connection.execute(f"DROP TABLE {missing}") + before = connection.execute("SELECT document FROM determa_timer_helpers").fetchall() + assert ( + helper.execute(reclaim(complete), principal="worker-A")["error_code"] + == "timer_capability_mismatch" + ) + with sqlite3.connect(helper.path) as connection: + assert connection.execute("SELECT document FROM determa_timer_helpers").fetchall() == before + assert connection.execute( + "SELECT COUNT(*) FROM sqlite_master WHERE name=?", (missing,) + ).fetchone() == (0,) From cb684360acb100653cacc711d9a74f9244a8850c Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 18:05:32 +0000 Subject: [PATCH 16/32] Bind native timer fate proof dispatch before checking overrides --- src/determa/state/timers.py | 5 ++++- tests/test_timer_native_fate.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/determa/state/timers.py b/src/determa/state/timers.py index 1404deb..b2f623c 100644 --- a/src/determa/state/timers.py +++ b/src/determa/state/timers.py @@ -896,7 +896,9 @@ def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, An if record["state"] == "claimed": if now < _time(record["expires_at"]): return _result(request, record, "timer_fire_in_progress") - self._prove_local_uncommitted_fire(connection, artifact, record) + # Dispatch through the retained implementation: an instance or + # class replacement must not bypass the checks inside the proof. + _FATE_PROOF(self, connection, artifact, record) if now < _time(record["deadline_at"]): return _result(request, record, "timer_not_due") expires_at = now + self.claim_lease_nanoseconds @@ -931,6 +933,7 @@ def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, An # Trusted local factory methods remain bound while an instance proves native fate. # This guard supplements native origin/history; it is not external registration proof. +_FATE_PROOF = SQLiteTimerHelper._prove_local_uncommitted_fire _FATE_METHODS = tuple( (name, getattr(SQLiteTimerHelper, name), getattr(SQLiteTimerHelper, name).__code__) for name in ( diff --git a/tests/test_timer_native_fate.py b/tests/test_timer_native_fate.py index 3dd6215..503f8df 100644 --- a/tests/test_timer_native_fate.py +++ b/tests/test_timer_native_fate.py @@ -67,6 +67,24 @@ def test_expired_local_claim_reclaims_with_new_fence_then_rejects_old_worker(tmp assert helper.execute(reclaim(complete), principal="worker-A") == result +@pytest.mark.parametrize("replace_class", [False, True]) +def test_replaced_fate_proof_cannot_bypass_its_own_guard(tmp_path, monkeypatch, replace_class): + helper, host, complete, _, now, _ = configured(tmp_path, replay_retention="bounded") + before_helper, before_root = helper.snapshot(), host.read_checkpoint("server-1").document + now[0] = "130" + monkeypatch.setattr( + SQLiteTimerHelper if replace_class else helper, + "_prove_local_uncommitted_fire", + lambda *_: None, + ) + assert ( + helper.execute(reclaim(complete), principal="worker-A")["error_code"] + == "delivery_ambiguous" + ) + assert helper.snapshot() == before_helper + assert host.read_checkpoint("server-1").document == before_root + + @pytest.mark.parametrize("changed_content", [False, True]) def test_independently_admitted_matching_id_never_becomes_helper_fate_proof( tmp_path, changed_content From fd90681c534707ba24826d02b0f962f3b1b84445 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 18:08:49 +0000 Subject: [PATCH 17/32] Attest retained timer proof code before native fate dispatch --- src/determa/state/timers.py | 5 +++++ tests/test_timer_native_fate.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/determa/state/timers.py b/src/determa/state/timers.py index b2f623c..2e9d8b6 100644 --- a/src/determa/state/timers.py +++ b/src/determa/state/timers.py @@ -898,6 +898,10 @@ def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, An return _result(request, record, "timer_fire_in_progress") # Dispatch through the retained implementation: an instance or # class replacement must not bypass the checks inside the proof. + # Function objects are mutable too: attest the retained code + # before entering it, rather than letting it attest itself. + if _FATE_PROOF.__code__ is not _FATE_PROOF_CODE: + raise TimerError("delivery_ambiguous") _FATE_PROOF(self, connection, artifact, record) if now < _time(record["deadline_at"]): return _result(request, record, "timer_not_due") @@ -934,6 +938,7 @@ def execute(self, request: Mapping[str, Any], *, principal: str) -> dict[str, An # Trusted local factory methods remain bound while an instance proves native fate. # This guard supplements native origin/history; it is not external registration proof. _FATE_PROOF = SQLiteTimerHelper._prove_local_uncommitted_fire +_FATE_PROOF_CODE = _FATE_PROOF.__code__ _FATE_METHODS = tuple( (name, getattr(SQLiteTimerHelper, name), getattr(SQLiteTimerHelper, name).__code__) for name in ( diff --git a/tests/test_timer_native_fate.py b/tests/test_timer_native_fate.py index 503f8df..b03f1bb 100644 --- a/tests/test_timer_native_fate.py +++ b/tests/test_timer_native_fate.py @@ -85,6 +85,24 @@ def test_replaced_fate_proof_cannot_bypass_its_own_guard(tmp_path, monkeypatch, assert host.read_checkpoint("server-1").document == before_root +@pytest.mark.parametrize("retention", ["bounded", "permanent"]) +def test_replaced_fate_function_code_is_checked_before_dispatch(tmp_path, monkeypatch, retention): + helper, host, complete, _, now, _ = configured(tmp_path, replay_retention=retention) + before_helper, before_root = helper.snapshot(), host.read_checkpoint("server-1").document + now[0] = "130" + monkeypatch.setattr( + SQLiteTimerHelper._prove_local_uncommitted_fire, + "__code__", + (lambda *_: None).__code__, + ) + assert ( + helper.execute(reclaim(complete), principal="worker-A")["error_code"] + == "delivery_ambiguous" + ) + assert helper.snapshot() == before_helper + assert host.read_checkpoint("server-1").document == before_root + + @pytest.mark.parametrize("changed_content", [False, True]) def test_independently_admitted_matching_id_never_becomes_helper_fate_proof( tmp_path, changed_content From b32ba125286a2c9b2dd96b48adcf5bf523f561de Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 18:56:09 +0000 Subject: [PATCH 18/32] Validate pinned native result incarnations before dispatch --- src/determa/state/effects.py | 16 ++++++- tests/test_committed_effects.py | 76 +++++++++++++++++++++++++++++---- 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 9691ce7..0c38ace 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -190,6 +190,13 @@ def _pinned_result_target( machine = definition["machine"] identity = [machine["namespace"], machine["machine_id"], machine["machine_version"]] if origin["kind"] == "root": + root_record = checkpoint["root_record"] + aggregate = root_record.get("aggregate_state") + actual_root_runtime_id = ( + aggregate["root_runtime_id"] if aggregate else root_record["root_runtime_id"] + ) + if origin["root_instance_id"] != root or runtime_id != actual_root_runtime_id: + raise EffectError("invalid_effect_journal") expected_id = hash_value( [ "determa-root-runtime-identity-1", @@ -308,6 +315,13 @@ def validate_journal( for item in checkpoint["pending_outbox_intents"] + checkpoint["terminal_outbox_records"] ] + [item["effect_id"] for item in checkpoint["outbox_effect_tombstones"]] for record in records: + # Validate every pinned incarnation before dispatch or native seeding, + # including nonterminal work. A self-consistent alternate trusted origin + # cannot replace the checkpoint's immutable root runtime identity. + try: + pinned_target = _pinned_result_target(checkpoint, record, resolver) + except (ArtifactError, KeyError, TypeError, ValueError) as exc: + raise EffectError("invalid_effect_journal") from exc intent = intents.get(record["effect_id"]) tombstone = compact.get(record["effect_id"]) if location_ids.count(record["effect_id"]) != 1 or record["intent_digest"] != ( @@ -373,7 +387,7 @@ def validate_journal( record, outcome["kind"], outcome["payload"], - _pinned_result_target(checkpoint, record, resolver), + pinned_target, ) receipt = record["admission_receipt"] if ( diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 6b26f25..3f48a43 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -157,6 +157,54 @@ def test_terminal_outcome_requires_immutable_attempt_evidence(): validate_journal(read("pending-checkpoint.json"), seal_journal(journal)) +def test_other_trusted_root_incarnation_refuses_before_native_seed(tmp_path): + """An independently resolvable origin is not this checkpoint's root identity.""" + from determa.state.wire import hash_value + + checkpoint = read("pending-checkpoint.json") + journal = read("data/leased-journal.json") + original = load_bundle((CASE / "machine.yaml").read_text()) + alternate = load_bundle( + (CASE / "machine.yaml") + .read_text() + .replace( + "namespace: conformance.committed_native_effects", + "namespace: conformance.other_trusted_definition", + ) + ) + resolver = MemoryArtifactResolver( + definitions={original.fingerprint: original, alternate.fingerprint: alternate} + ) + record = journal["effect_records"][0] + origin = record["target"]["runtime_incarnation"] + definition = origin["definition"] + definition["validated_bundle_fingerprint"] = alternate.fingerprint + definition["machine"]["namespace"] = alternate.namespace + machine = definition["machine"] + record["target"]["runtime_id"] = hash_value( + [ + "determa-root-runtime-identity-1", + "1", + alternate.fingerprint, + machine["namespace"], + machine["machine_id"], + machine["machine_version"], + checkpoint["root_instance_id"], + ] + ) + journal = seal_journal(journal) + with pytest.raises(EffectError, match="invalid_effect_journal"): + validate_journal(checkpoint, journal, resolver) + host = SQLiteCommittedEffectHost(tmp_path / "invalid-seed.sqlite", resolver, {}, None) + host.setup_schema() + with pytest.raises(EffectError, match="invalid_effect_journal"): + host.seed(checkpoint, journal) + with host._connect() as connection: + assert ( + connection.execute("SELECT COUNT(*) FROM determa_committed_effects").fetchone()[0] == 0 + ) + + @pytest.mark.parametrize("field", ["digest", "attempt_fence"]) def test_terminal_outcome_rejects_forged_evidence(field): journal = copy.deepcopy(read("data/outcome-recorded-journal.json")) @@ -1537,6 +1585,8 @@ def test_native_producer_replay_retains_exact_kind_and_request_identity(tmp_path def test_portable_producer_identity_is_not_inferred_from_result_destination(tmp_path): + from determa.state.effects import _reconstruct_producer_response + original, root, _, _ = host_fixture(tmp_path) checkpoint = read("pending-checkpoint.json") journal = read("data/unclaimed-journal.json") @@ -1547,15 +1597,23 @@ def test_portable_producer_identity_is_not_inferred_from_result_destination(tmp_ tmp_path / "different-target.sqlite", original.resolver, {}, None ) host.setup_schema() - host.seed(checkpoint, seal_journal(journal)) - arguments = { - "expected_revision": "stale", - "expected_digest": "stale", - "route_generation": "999", + journal = seal_journal(journal) + # This deliberately inconsistent destination must not become native work. + # The pure receipt reconstruction still identifies its producing operation + # from retained checkpoint evidence, rather than the result destination. + with pytest.raises(EffectError, match="invalid_effect_journal"): + host.seed(checkpoint, journal) + identity = { + "operation_kind": "produce", + "root_instance_id": root, + "target_runtime_id": source_runtime, "operation_token": record["operation_token"], } - assert host.produce(root, "produce-1", source_runtime, **arguments) == read( - "data/producer-response.json" - ) + assert _reconstruct_producer_response( + checkpoint, journal, "produce-1", identity, original.resolver + ) == read("data/producer-response.json") + identity["target_runtime_id"] = record["target"]["runtime_id"] with pytest.raises(EffectError, match="replay_evidence_expired"): - host.produce(root, "produce-1", record["target"]["runtime_id"], **arguments) + _reconstruct_producer_response( + checkpoint, journal, "produce-1", identity, original.resolver + ) From 67f3ca696276143ba63937a98fb83f57564db111 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 19:09:11 +0000 Subject: [PATCH 19/32] Keep cancelled and terminal native work unclaimable --- src/determa/state/effects.py | 40 ++++++++++++++- tests/test_committed_effects.py | 90 +++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 2 deletions(-) diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 0c38ace..9810862 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -332,9 +332,44 @@ def validate_journal( raise EffectError("invalid_effect_journal") reports = record["attempt_records"] fences = [int(item["attempt_fence"]) for item in reports] - if fences != sorted(set(fences)) or any(f > int(record["attempt_fence"]) for f in fences): + if fences != sorted(set(fences)) or any( + f == 0 or f > int(record["attempt_fence"]) for f in fences + ): raise EffectError("invalid_effect_journal") state = record["invocation_state"] + outcome = record["outcome"] + current_report = next( + (item for item in reports if item["attempt_fence"] == record["attempt_fence"]), + None, + ) + if state in {"leased", "ambiguous"} and record["attempt_fence"] == "0": + raise EffectError("invalid_effect_journal") + if state in {"unclaimed", "leased", "ambiguous"} and any( + item["report_kind"] in _OUTCOMES for item in reports + ): + # A committed terminal report cannot become retryable work merely + # because its outcome/result fields were stripped. + raise EffectError("invalid_effect_journal") + if current_report is not None and ( + state == "leased" + or (state == "unclaimed" and current_report["report_kind"] != "retryable_failure") + or (state == "ambiguous" and current_report["report_kind"] != "ambiguous") + ): + raise EffectError("invalid_effect_journal") + cancellation = record["cancellation"] + if ( + cancellation is not None + and cancellation["state"] == "prevented_start" + and ( + state not in {"outcome_recorded", "result_admitted", "closed"} + or record["attempt_fence"] != "0" + or reports + or outcome is None + or outcome["kind"] != "cancelled" + or outcome["attempt_fence"] != "0" + ) + ): + raise EffectError("invalid_effect_journal") if state in {"unclaimed", "leased", "ambiguous"} and any( record[key] is not None for key in ("outcome", "result_event_id", "admission_receipt") ): @@ -351,8 +386,9 @@ def validate_journal( or record["admission_receipt"] not in checkpoint["operation_receipts"] ): raise EffectError("invalid_effect_journal") - outcome = record["outcome"] if outcome is not None: + if outcome["attempt_fence"] != record["attempt_fence"]: + raise EffectError("invalid_effect_journal") if outcome["digest"] != hash_value( [ "determa-effect-outcome-1", diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 3f48a43..8a329ad 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -205,6 +205,96 @@ def test_other_trusted_root_incarnation_refuses_before_native_seed(tmp_path): ) +@pytest.mark.parametrize("state", ["unclaimed", "leased", "ambiguous"]) +def test_prevented_start_without_cancelled_outcome_cannot_be_native_work(tmp_path, state): + checkpoint = read("pending-checkpoint.json") + journal = read("data/unclaimed-journal.json") + record = journal["effect_records"][0] + record["invocation_state"] = state + record["attempt_fence"] = "0" if state == "unclaimed" else "1" + record["cancellation"] = { + "operation_id": "cancel-before-claim", + "reason": "user_requested", + "state": "prevented_start", + } + _assert_invalid_native_seed(tmp_path, checkpoint, seal_journal(journal)) + + +@pytest.mark.parametrize("state", ["unclaimed", "leased", "ambiguous"]) +def test_stripping_terminal_outcome_never_makes_native_work_claimable(tmp_path, state): + checkpoint = read("pending-checkpoint.json") + journal = read("data/outcome-recorded-journal.json") + journal["effect_records"][0].update( + invocation_state=state, outcome=None, result_event_id=None, admission_receipt=None + ) + _assert_invalid_native_seed(tmp_path, checkpoint, seal_journal(journal)) + + +def _assert_invalid_native_seed(tmp_path, checkpoint, journal): + with pytest.raises(EffectError, match="invalid_effect_journal"): + validate_journal(checkpoint, journal) + bundle = load_bundle((CASE / "machine.yaml").read_text()) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + fresh = SQLiteCommittedEffectHost(tmp_path / "bad-seed.sqlite", resolver, {}, None) + fresh.setup_schema() + with pytest.raises(EffectError, match="invalid_effect_journal"): + fresh.seed(checkpoint, journal) + with fresh._connect() as connection: + assert ( + connection.execute("SELECT COUNT(*) FROM determa_committed_effects").fetchone()[0] == 0 + ) + + +@pytest.mark.parametrize( + ("state", "report_kind", "report_fence"), + [ + ("unclaimed", "retryable_failure", "0"), + ("unclaimed", "ambiguous", "1"), + ("ambiguous", "retryable_failure", "1"), + ("leased", "retryable_failure", "1"), + ], +) +def test_completed_attempt_report_cannot_disagree_with_native_invocation_state( + state, report_kind, report_fence +): + from determa.state.wire import hash_value + + checkpoint = read("pending-checkpoint.json") + journal = read("data/leased-journal.json") + record = journal["effect_records"][0] + reason = ( + "no_call_proven" if report_kind == "retryable_failure" else "provider_acceptance_unknown" + ) + record["invocation_state"] = state + record["attempt_records"] = [ + { + "attempt_fence": report_fence, + "report_kind": report_kind, + "report_digest": hash_value( + [ + "determa-effect-attempt-report-1", + record["effect_id"], + record["operation_token"], + report_fence, + report_kind, + ["map", []], + reason, + ] + ), + "reason": reason, + } + ] + with pytest.raises(EffectError, match="invalid_effect_journal"): + validate_journal(checkpoint, seal_journal(journal)) + + +def test_terminal_outcome_cannot_supersede_a_newer_attempt_fence(): + journal = read("data/outcome-recorded-journal.json") + journal["effect_records"][0]["attempt_fence"] = "2" + with pytest.raises(EffectError, match="invalid_effect_journal"): + validate_journal(read("pending-checkpoint.json"), seal_journal(journal)) + + @pytest.mark.parametrize("field", ["digest", "attempt_fence"]) def test_terminal_outcome_rejects_forged_evidence(field): journal = copy.deepcopy(read("data/outcome-recorded-journal.json")) From 9dfc9af2765715cd10b58d20edf4587a4cf3b6a4 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 19:24:08 +0000 Subject: [PATCH 20/32] Reject missing retry evidence and superseded terminal attempts --- src/determa/state/effects.py | 9 +++++++ tests/test_committed_effects.py | 44 +++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 9810862..b5f7fa9 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -342,6 +342,15 @@ def validate_journal( (item for item in reports if item["attempt_fence"] == record["attempt_fence"]), None, ) + if state == "unclaimed" and record["attempt_fence"] != "0" and current_report is None: + # A consumed attempt cannot become new work without retaining the + # current safe-failure report. Missing evidence is not retry proof. + raise EffectError("invalid_effect_journal") + terminal_reports = [item for item in reports if item["report_kind"] in _OUTCOMES] + if len(terminal_reports) > 1: + # Terminal outcomes are immutable: no later attempt may supersede + # an earlier committed terminal report, even with valid hashes. + raise EffectError("invalid_effect_journal") if state in {"leased", "ambiguous"} and record["attempt_fence"] == "0": raise EffectError("invalid_effect_journal") if state in {"unclaimed", "leased", "ambiguous"} and any( diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 8a329ad..14208c0 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -295,6 +295,45 @@ def test_terminal_outcome_cannot_supersede_a_newer_attempt_fence(): validate_journal(read("pending-checkpoint.json"), seal_journal(journal)) +def test_positive_fence_unclaimed_requires_current_safe_retry_report(tmp_path): + journal = read("data/leased-journal.json") + journal["effect_records"][0]["invocation_state"] = "unclaimed" + _assert_invalid_native_seed(tmp_path, read("pending-checkpoint.json"), seal_journal(journal)) + + +def test_terminal_outcome_cannot_follow_an_earlier_terminal_attempt(tmp_path): + from determa.state.wire import hash_value + + journal = read("data/outcome-recorded-journal.json") + record = journal["effect_records"][0] + outcome = record["outcome"] + report = copy.deepcopy(record["attempt_records"][0]) + record["attempt_fence"] = outcome["attempt_fence"] = report["attempt_fence"] = "2" + report["report_digest"] = hash_value( + [ + "determa-effect-attempt-report-1", + record["effect_id"], + record["operation_token"], + "2", + outcome["kind"], + outcome["payload"], + report["reason"], + ] + ) + outcome["digest"] = hash_value( + [ + "determa-effect-outcome-1", + record["effect_id"], + record["operation_token"], + outcome["kind"], + outcome["payload"], + "2", + ] + ) + record["attempt_records"].append(report) + _assert_invalid_native_seed(tmp_path, read("pending-checkpoint.json"), seal_journal(journal)) + + @pytest.mark.parametrize("field", ["digest", "attempt_fence"]) def test_terminal_outcome_rejects_forged_evidence(field): journal = copy.deepcopy(read("data/outcome-recorded-journal.json")) @@ -711,6 +750,11 @@ def test_effect_mutations_refuse_a_torn_authority_journal_pair(tmp_path, changed else: document = host.snapshot(root) document["journal"]["effect_records"][0]["attempt_fence"] = "1" + # Keep the journal internally valid so this exercises the separate + # native authority/journal fence mismatch, not missing retry evidence. + document["journal"]["effect_records"][0]["attempt_records"] = read( + "data/retryable-journal.json" + )["effect_records"][0]["attempt_records"] document["journal"] = seal_journal(document["journal"]) with host._connect() as connection: connection.execute( From e53809e4b0d95617d0742ef80616cca5c78b9936 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 21:35:31 +0000 Subject: [PATCH 21/32] Retain native dispatch starts and verify call boundaries (WIP) --- src/determa/state/effects.py | 162 +++++++++++- tests/test_committed_effects.py | 286 +++++++++++++++++++++- tests/test_effect_dispatch_native_fate.py | 224 +++++++++++++++++ 3 files changed, 658 insertions(+), 14 deletions(-) create mode 100644 tests/test_effect_dispatch_native_fate.py diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index b5f7fa9..a73dd73 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -686,6 +686,36 @@ def visit(value: Any, pointer: str) -> None: return response +def _native_effect_journal(ledger: Mapping[str, Any], root: str) -> dict[str, Any]: + """Resolve only host-participant history, never generic proposed mutation bytes.""" + from .authority import _parse + + history = ledger.get("native_effect_journal_bytes") + if type(history) is not list or any(type(source) is not str for source in history): + raise EffectError("unauthorized_scope") + latest = None + for source in history: + try: + journal = _parse(source) + if type(journal) is not dict: + raise EffectError("unauthorized_scope") + if ( + canonical_bytes(journal).decode() != source + or journal.get("host_effect_journal_format") != "determa.host_effect_journal" + or journal.get("host_effect_journal_schema_version") != 1 + or journal.get("scope_identity") != ledger["scope_identity"] + or journal.get("host_effect_journal_digest") != journal_digest(journal) + ): + raise EffectError("unauthorized_scope") + except (ValueError, TypeError, KeyError) as error: + raise EffectError("unauthorized_scope") from error + if journal["root_instance_id"] == root: + latest = journal + if latest is None: + raise EffectError("unauthorized_scope") + return latest + + def _validate_authority_pair(ledger: Mapping[str, Any], document: Mapping[str, Any]) -> None: """Check the old native pair before any journal transition can be staged.""" from .authority import _native_checkpoint_history @@ -740,6 +770,8 @@ def _validate_authority_pair(ledger: Mapping[str, Any], document: Mapping[str, A raise EffectError("stale_attempt_fence") elif live is not None: raise EffectError("stale_attempt_fence") + if journal != _native_effect_journal(ledger, journal["root_instance_id"]): + raise EffectError("unauthorized_scope") def _issue_effect_claim( @@ -857,11 +889,28 @@ def invoke( payload: Any, metadata: Mapping[str, Any], attempt: Mapping[str, Any], + ) -> Mapping[str, Any]: + return VerifiedNativeHandler._invoke_with_guard( + self, reference, payload, metadata, attempt, lambda: None + ) + + def _invoke_with_guard( + self, + reference: Mapping[str, Any], + payload: Any, + metadata: Mapping[str, Any], + attempt: Mapping[str, Any], + guard: Callable[[], None], ) -> Mapping[str, Any]: method, instance = VerifiedNativeHandler.verify( self, reference, metadata["destination_binding_digest"] ) - return cast(Mapping[str, Any], method(instance, payload, metadata, attempt)) + # Provider verification may execute native callbacks. Live host rights and + # trusted time must be checked after those callbacks, immediately before I/O. + guard() + result = method(instance, payload, metadata, attempt) + VerifiedNativeHandler.verify(self, reference, metadata["destination_binding_digest"]) + return cast(Mapping[str, Any], result) class SQLiteCommittedEffectHost: @@ -963,6 +1012,10 @@ def seed( or journal["scope_identity"] != self.authority_scope ): raise EffectError("unauthorized_scope") + if checkpoint["root_instance_id"] in ledger.get( + "native_effect_roots", [] + ) and journal != _native_effect_journal(ledger, checkpoint["root_instance_id"]): + raise EffectError("unauthorized_scope") # Restoring an already claimed invocation cannot mint authority. # Its exact claim must already have been issued by this ledger. if claim is not None and ( @@ -1085,6 +1138,11 @@ def _mirror_authority( history.append(source) ledger["checkpoint_bytes"].append(source) ledger["mutation_bytes"].append(canonical_bytes(document["journal"]).decode()) + # This private native role history cannot be fabricated by a generic + # guarded_commit payload that happens to resemble a journal. + ledger.setdefault("native_effect_journal_bytes", []).append( + canonical_bytes(document["journal"]).decode() + ) if advance_generation: ledger["scope_generation"] = str(int(ledger["scope_generation"]) + 1) @@ -1539,8 +1597,94 @@ def dispatch( if clock is None: raise EffectError("stale_attempt_fence") + def start(document: dict[str, Any]) -> str: + record = _record(document["journal"], effect_id) + if record["invocation_state"] != "leased": + raise EffectError("effect_not_outstanding") + self._authorize( + document, + effect_id, + record["operation_token"], + record["attempt_fence"], + principal, + scope, + epoch, + clock(), + ) + if ( + not authorized + or credential is None + or record["handler_reference"] != self.route["handler_reference"] + or record["destination_binding_digest"] != self.route["destination_binding_digest"] + ): + raise EffectError("unauthorized_scope") + self._verified_handler( + record["handler_reference"], record["destination_binding_digest"] + ) + identifier = hash_value( + [ + "determa-native-invocation-start-1", + scope, + root, + effect_id, + record["attempt_fence"], + ] + ) + journal = document["journal"] + if identifier in document["responses"] or any( + reference["operation_id"] == identifier + for reference in journal["operation_response_references"] + ): + raise EffectError("native_invocation_already_started") + response = { + "kind": "effect_invocation_start", + "body": {"claim": copy.deepcopy(document["claims"][effect_id])}, + } + document["responses"][identifier] = response + journal["operation_response_references"].append( + { + "operation_id": identifier, + "response_digest": hash_value(["determa-host-operation-response-1", response]), + } + ) + journal["operation_response_references"].sort( + key=lambda reference: reference["operation_id"] + ) + _bump(journal) + return identifier + + # Only this successful new native commit reaches the call phase. A saved + # marker is historical evidence, never permission for another invocation. + start_id = self._transact( + root, + start, + expected_epoch=epoch, + authority_mutation=self._mirror_authority, + worker_guard=(effect_id, principal, None, trusted_now), + ) + def call(document: dict[str, Any]) -> Mapping[str, Any]: record = _record(document["journal"], effect_id) + response = document["responses"].get(start_id) + reference = next( + ( + item + for item in document["journal"]["operation_response_references"] + if item["operation_id"] == start_id + ), + None, + ) + if ( + response + != { + "kind": "effect_invocation_start", + "body": {"claim": document["claims"].get(effect_id)}, + } + or reference is None + or reference["response_digest"] + != hash_value(["determa-host-operation-response-1", response]) + ): + raise EffectError("invalid_effect_journal") if record["invocation_state"] != "leased": raise EffectError("effect_not_outstanding") self._authorize( @@ -1580,12 +1724,26 @@ def call(document: dict[str, Any]) -> Mapping[str, Any]: handler = self._verified_handler( record["handler_reference"], record["destination_binding_digest"] ) - result = VerifiedNativeHandler.invoke( + + def live_call_guard() -> None: + self._authorize( + document, + effect_id, + record["operation_token"], + record["attempt_fence"], + principal, + scope, + epoch, + clock(), + ) + + result = VerifiedNativeHandler._invoke_with_guard( handler, record["handler_reference"], copy.deepcopy(intent["payload"]), metadata, {"attempt_fence": record["attempt_fence"]}, + live_call_guard, ) # External acceptance cannot extend an expired worker's mutation # rights. A refusal leaves its leased journal for reconciliation. diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 14208c0..cb3cacc 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -92,7 +92,12 @@ def verify(source, actual): ) handler = VerifiedNativeHandler(registry, configured) return SQLiteCommittedEffectHost( - host.path, host.resolver, host.route, handler, trusted_clock=host.trusted_clock + host.path, + host.resolver, + host.route, + handler, + authority_scope=host.authority_scope, + trusted_clock=host.trusted_clock, ) @@ -803,27 +808,36 @@ def test_authority_claim_refuses_missing_native_participant_evidence(tmp_path, d def test_seed_preserves_historical_claim_without_reviving_it(tmp_path): authority, host, scope, root, record = authority_effect_fixture(tmp_path) - request, invocation = authority_claim_request(authority, scope, root, record) - claim = json.loads(authority.perform(request, invocation))["claim"] - # Model a retained completed attempt whose current journal permits a retry. - journal = read("data/retryable-journal.json") + command, invocation = authority_claim_request(authority, scope, root, record) + claim = json.loads(authority.perform(command, invocation))["claim"] + # Use the actual committed report transition rather than fabricating an older + # portable journal and editing native authority to make it look admissible. + report = { + "effect_id": record["effect_id"], + "operation_token": record["operation_token"], + "attempt_fence": claim["attempt_fence"], + "outcome_kind": "retryable_failure", + "payload": ["map", []], + } + host.trusted_clock = lambda: "0" + response = host.submit_result( + root, report, principal=claim["worker_principal"], scope=scope, epoch="0", trusted_now="0" + ) + assert response["status"] == "report_recorded" + saved = host.snapshot(root) ledger = authority.inspect(scope) - ledger["active_claims"] = [] + assert ledger["active_claims"] == [] assert claim in ledger["effect_claim_history"] - checkpoint = host.snapshot(root)["checkpoint"] with authority._connect() as connection: - connection.execute("UPDATE determa_scope_authority SET ledger=?", (json.dumps(ledger),)) connection.execute("DELETE FROM determa_committed_effects") connection.commit() - host.seed(checkpoint, journal, claim) + host.seed(saved["checkpoint"], saved["journal"], claim) assert authority.inspect(scope)["active_claims"] == [] assert host.snapshot(root)["claims"][record["effect_id"]] == claim new_claim = host.claim( root, record["effect_id"], "worker-b", "0", expires_at="30", trusted_now="20" ) assert new_claim["attempt_fence"] == "2" - assert authority.inspect(scope)["active_claims"] == [new_claim] - assert claim in authority.inspect(scope)["effect_claim_history"] def test_missing_native_work_cannot_be_rebound_to_another_authorized_root(tmp_path): @@ -959,7 +973,17 @@ def external_acceptance(*args): with pytest.raises(EffectError, match="stale_attempt_fence"): host.dispatch(root, request["effect_id"], credential="test-credential", **context) assert len(calls) == 1 - assert host.snapshot(root) == before + after = host.snapshot(root) + assert after["checkpoint"] == before["checkpoint"] + assert after["journal"]["effect_records"] == before["journal"]["effect_records"] + assert ( + int(after["journal"]["journal_revision"]) == int(before["journal"]["journal_revision"]) + 1 + ) + assert len(after["responses"]) == len(before["responses"]) + 1 + current_time[0] = "0" + with pytest.raises(EffectError, match="native_invocation_already_started"): + host.dispatch(root, request["effect_id"], credential="test-credential", **context) + assert len(calls) == 1 assert host.recover(root)["journal"]["effect_records"][0]["invocation_state"] == "ambiguous" @@ -1751,3 +1775,241 @@ def test_portable_producer_identity_is_not_inferred_from_result_destination(tmp_ _reconstruct_producer_response( checkpoint, journal, "produce-1", identity, original.resolver ) + + +def test_dispatch_retained_start_denies_repeated_and_restarted_calls(tmp_path): + host, root, request, context = host_fixture(tmp_path) + record = host.snapshot(root)["journal"]["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + calls = [] + host = installed_test_handler(host, lambda *args: calls.append(args) or {"accepted": True}) + assert host.dispatch(root, request["effect_id"], credential="test-credential", **context) == { + "accepted": True + } + after = host.snapshot(root) + assert len(calls) == 1 + with pytest.raises(EffectError, match="native_invocation_already_started"): + host.dispatch(root, request["effect_id"], credential="test-credential", **context) + reopened = SQLiteCommittedEffectHost( + host.path, host.resolver, host.route, host.handler, trusted_clock=lambda: "0" + ) + with pytest.raises(EffectError, match="native_invocation_already_started"): + reopened.dispatch(root, request["effect_id"], credential="test-credential", **context) + assert reopened.snapshot(root) == after + assert len(calls) == 1 + + +def test_dispatch_provider_error_retains_start_and_cannot_be_retried_by_reopening(tmp_path): + host, root, request, context = host_fixture(tmp_path) + before = host.snapshot(root) + record = before["journal"]["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + calls = [] + + def failed_call(*args): + calls.append(args) + raise RuntimeError("external acceptance is unknown") + + host = installed_test_handler(host, failed_call) + with pytest.raises(RuntimeError, match="acceptance is unknown"): + host.dispatch(root, request["effect_id"], credential="test-credential", **context) + after = host.snapshot(root) + assert after["checkpoint"] == before["checkpoint"] + assert after["journal"]["effect_records"] == before["journal"]["effect_records"] + reopened = SQLiteCommittedEffectHost( + host.path, host.resolver, host.route, host.handler, trusted_clock=lambda: "0" + ) + with pytest.raises(EffectError, match="native_invocation_already_started"): + reopened.dispatch(root, request["effect_id"], credential="test-credential", **context) + recovered = reopened.recover(root) + assert recovered["journal"]["effect_records"][0]["invocation_state"] == "ambiguous" + assert len(calls) == 1 + + +def test_dispatch_clock_is_rechecked_after_provider_verification_before_io(tmp_path, monkeypatch): + import sqlite3 + + host, root, request, context = host_fixture(tmp_path) + before = host.snapshot(root) + record = before["journal"]["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + current_time = ["0"] + expiry = read("data/active-claim.json")["expires_at"] + + def health(self, instance): + with sqlite3.connect(host.path) as connection: + document = json.loads( + connection.execute("SELECT document FROM determa_committed_effects").fetchone()[0] + ) + if any( + response.get("kind") == "effect_invocation_start" + for response in document["responses"].values() + ): + current_time[0] = expiry + return "healthy" + + monkeypatch.setattr(NativeTestProvider, "health", health) + calls = [] + host.trusted_clock = lambda: current_time[0] + host = installed_test_handler(host, lambda *args: calls.append(args) or {}) + with pytest.raises(EffectError, match="stale_attempt_fence"): + host.dispatch(root, request["effect_id"], credential="test-credential", **context) + after = host.snapshot(root) + assert after["checkpoint"] == before["checkpoint"] + assert after["journal"]["effect_records"] == before["journal"]["effect_records"] + assert len(after["responses"]) == 1 + assert calls == [] + + +def test_dispatch_postcall_health_loss_retains_start_without_returning_verified_output( + tmp_path, monkeypatch +): + host, root, request, context = host_fixture(tmp_path) + record = host.snapshot(root)["journal"]["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + healthy = [True] + monkeypatch.setattr( + NativeTestProvider, "health", lambda *args: "healthy" if healthy[0] else "unavailable" + ) + calls = [] + + def external_acceptance(*args): + calls.append(args) + healthy[0] = False + return {"accepted": True} + + host = installed_test_handler(host, external_acceptance) + with pytest.raises(EffectError, match="host_capability_mismatch"): + host.dispatch(root, request["effect_id"], credential="test-credential", **context) + after = host.snapshot(root) + healthy[0] = True + with pytest.raises(EffectError, match="native_invocation_already_started"): + host.dispatch(root, request["effect_id"], credential="test-credential", **context) + assert host.snapshot(root) == after + assert len(calls) == 1 + + +def test_identical_dispatch_contenders_issue_one_native_call_under_consuming_lock(tmp_path): + import sqlite3 + import threading + from concurrent.futures import ThreadPoolExecutor + + host, root, request, context = host_fixture(tmp_path) + record = host.snapshot(root)["journal"]["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + entered, release = threading.Event(), threading.Event() + calls = [] + + def native_call(*args): + calls.append(args) + entered.set() + assert release.wait(5) + return {"accepted": True} + + host = installed_test_handler(host, native_call) + rendezvous = threading.Barrier(3) + + def contender(): + rendezvous.wait(timeout=5) + try: + return host.dispatch( + root, request["effect_id"], credential="test-credential", **context + ) + except EffectError as error: + return str(error) + + with ThreadPoolExecutor(max_workers=2) as workers: + futures = [workers.submit(contender) for _ in range(2)] + rendezvous.wait(timeout=5) + try: + assert entered.wait(5) + with sqlite3.connect(host.path, timeout=0, isolation_level=None) as writer: + with pytest.raises(sqlite3.OperationalError, match="locked"): + writer.execute("BEGIN IMMEDIATE") + assert len(calls) == 1 + finally: + release.set() + replies = [future.result(timeout=5) for future in futures] + assert replies.count({"accepted": True}) == 1 + assert replies.count("native_invocation_already_started") == 1 + assert len(calls) == 1 + + +def test_dispatch_native_call_excludes_a_real_competing_result_transaction(tmp_path, monkeypatch): + import sqlite3 + + host, root, request, context = host_fixture(tmp_path) + record = host.snapshot(root)["journal"]["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + other = SQLiteCommittedEffectHost( + host.path, host.resolver, host.route, None, trusted_clock=lambda: "0" + ) + native_connect = other._connect + + def without_waiting(): + connection = native_connect() + connection.execute("PRAGMA busy_timeout=0") + return connection + + monkeypatch.setattr(other, "_connect", without_waiting) + calls = [] + + def native_call(*args): + calls.append(args) + with pytest.raises(sqlite3.OperationalError, match="locked"): + other.submit_result(root, request, **context) + return {"accepted": True} + + host = installed_test_handler(host, native_call) + assert host.dispatch(root, request["effect_id"], credential="test-credential", **context) == { + "accepted": True + } + after = host.snapshot(root) + assert after["journal"]["effect_records"][0]["invocation_state"] == "leased" + assert after["journal"]["effect_records"][0]["outcome"] is None + assert len(calls) == 1 + assert other.submit_result(root, request, **context)["status"] == "committed" + + +def test_native_authority_seed_cannot_erase_a_committed_dispatch_start(tmp_path): + import sqlite3 + + authority, host, scope, root, record = authority_effect_fixture(tmp_path) + command, invocation = authority_claim_request(authority, scope, root, record) + assert json.loads(authority.perform(command, invocation))["status"] == "accepted" + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + host.trusted_clock = lambda: "0" + calls = [] + host = installed_test_handler(host, lambda *args: calls.append(args) or {"accepted": True}) + before = host.snapshot(root) + claim = before["claims"][record["effect_id"]] + context = { + "principal": claim["worker_principal"], + "scope": scope, + "epoch": "0", + "trusted_now": "0", + } + host.dispatch(root, record["effect_id"], credential="test-credential", **context) + ledger = authority.inspect(scope) + with sqlite3.connect(host.path) as connection: + connection.execute( + "DELETE FROM determa_committed_effects WHERE root_instance_id=?", (root,) + ) + with pytest.raises(EffectError, match="unauthorized_scope"): + host.seed(before["checkpoint"], before["journal"], claim) + assert authority.inspect(scope) == ledger + assert len(calls) == 1 diff --git a/tests/test_effect_dispatch_native_fate.py b/tests/test_effect_dispatch_native_fate.py new file mode 100644 index 0000000..dfd0c05 --- /dev/null +++ b/tests/test_effect_dispatch_native_fate.py @@ -0,0 +1,224 @@ +"""SIGKILL retains native start/outcome facts across actual independent SQLite I/O.""" + +from __future__ import annotations + +import copy +import json +import multiprocessing +import os +import signal +import sqlite3 +import time +from pathlib import Path + +import pytest + +from determa.state import MemoryArtifactResolver, load_bundle +from determa.state.effects import EffectError, SQLiteCommittedEffectHost + +from .test_committed_effects import ( + CASE, + NativeTestProvider, + authority_claim_request, + authority_effect_fixture, + installed_test_handler, +) + + +def _dispatch_worker(directory, cut, control): + authority, host, scope, root, record = authority_effect_fixture(Path(directory)) + command, invocation = authority_claim_request(authority, scope, root, record) + claim = json.loads(authority.perform(command, invocation))["claim"] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + host.trusted_clock = lambda: "0" + context = { + "principal": claim["worker_principal"], + "scope": scope, + "epoch": "0", + "trusted_now": "0", + } + information = {"root": root, "scope": scope, "effect_id": record["effect_id"], "claim": claim} + provider_path = Path(directory) / "provider.sqlite" + + def health(self, instance): + if cut == "start": + with sqlite3.connect(host.path) as connection: + document = json.loads( + connection.execute("SELECT document FROM determa_committed_effects").fetchone()[ + 0 + ] + ) + if any( + response.get("kind") == "effect_invocation_start" + for response in document["responses"].values() + ): + control.send(information) + time.sleep(60) + return "healthy" + + NativeTestProvider.health = health + + def native_call(payload, metadata, attempt): + # SDK objects stay in this native callback. The independent destination + # transaction commits before acceptance is reported to the host. + class SdkReceipt: + def __init__(self): + self.reference = "provider:" + metadata["effect_id"] + + receipt = SdkReceipt() + evidence = { + "scope_identity": metadata["scope_identity"], + "effect_id": metadata["effect_id"], + "operation_token": metadata["operation_token"], + "attempt_fence": attempt["attempt_fence"], + "request": payload, + "provider_reference": receipt.reference, + } + assert metadata["credential"] == "test-credential" + with sqlite3.connect(provider_path) as destination: + destination.execute("PRAGMA journal_mode=WAL") + destination.execute("PRAGMA synchronous=FULL") + destination.execute( + "CREATE TABLE receipts (scope TEXT NOT NULL, effect TEXT NOT NULL, " + "receipt TEXT NOT NULL, PRIMARY KEY(scope,effect))" + ) + destination.execute( + "INSERT INTO receipts VALUES (?,?,?)", + (scope, record["effect_id"], json.dumps(evidence, sort_keys=True)), + ) + destination.commit() + if cut == "acceptance": + control.send(information) + time.sleep(60) + return { + "report_kind": "succeeded", + "payload": ["map", [["provider_reference", ["string", receipt.reference]]]], + "reason": None, + } + + host = installed_test_handler(host, native_call) + reply = host.dispatch(root, record["effect_id"], credential="test-credential", **context) + if cut == "response": + control.send(information) + time.sleep(60) + + def outcome_committed(*args): + control.send(information) + time.sleep(60) + + if cut == "outcome": + host.core_observer = outcome_committed + request = { + "effect_id": record["effect_id"], + "operation_token": record["operation_token"], + "attempt_fence": claim["attempt_fence"], + "outcome_kind": reply["report_kind"], + "payload": reply["payload"], + } + response = host.submit_result(root, request, **context) + assert cut == "admission" and response["status"] == "committed" + information["response"] = response + control.send(information) + time.sleep(60) + + +@pytest.mark.skipif(os.name != "posix", reason="requires actual SIGKILL") +@pytest.mark.parametrize("cut", ["start", "acceptance", "response", "outcome", "admission"]) +def test_sigkill_native_dispatch_never_repeats_uncertain_provider_call(tmp_path, cut): + context = multiprocessing.get_context("spawn") + parent, child = context.Pipe() + worker = context.Process(target=_dispatch_worker, args=(str(tmp_path), cut, child)) + worker.start() + child.close() + try: + assert parent.poll(20), f"worker did not reach {cut} (exit={worker.exitcode})" + information = parent.recv() + worker.kill() + worker.join(5) + assert worker.exitcode == -signal.SIGKILL + finally: + if worker.is_alive(): + worker.kill() + worker.join(5) + parent.close() + bundle = load_bundle((CASE / "machine.yaml").read_text()) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + route = {"authority_epoch": "0"} + base = SQLiteCommittedEffectHost( + tmp_path / "paired.sqlite", + resolver, + route, + None, + authority_scope=information["scope"], + trusted_clock=lambda: "0", + ) + before = base.snapshot(information["root"]) + record = before["journal"]["effect_records"][0] + base.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + calls = [] + host = installed_test_handler(base, lambda *args: calls.append(args) or {}) + invocation = { + "principal": information["claim"]["worker_principal"], + "scope": information["scope"], + "epoch": "0", + "trusted_now": "0", + } + with pytest.raises(EffectError): + host.dispatch( + information["root"], + information["effect_id"], + credential="test-credential", + **invocation, + ) + assert host.snapshot(information["root"]) == before + assert calls == [] + assert ( + len( + [ + reply + for reply in before["responses"].values() + if reply.get("kind") == "effect_invocation_start" + ] + ) + == 1 + ) + assert record["invocation_state"] == ( + "result_admitted" + if cut == "admission" + else "outcome_recorded" + if cut == "outcome" + else "leased" + ) + if cut == "start": + assert not (tmp_path / "provider.sqlite").exists() + else: + with sqlite3.connect(tmp_path / "provider.sqlite") as destination: + rows = destination.execute("SELECT scope,effect,receipt FROM receipts").fetchall() + assert len(rows) == 1 + scope, effect, encoded = rows[0] + receipt = json.loads(encoded) + assert (scope, effect) == (information["scope"], information["effect_id"]) + assert receipt["operation_token"] == record["operation_token"] + assert receipt["attempt_fence"] == "1" + assert ( + receipt["request"] + == before["checkpoint"]["pending_outbox_intents"][0]["intent"]["payload"] + ) + host.trusted_clock = lambda: information["claim"]["expires_at"] + recovered = host.recover(information["root"]) + assert calls == [] + assert recovered["journal"]["effect_records"][0]["invocation_state"] == ( + "result_admitted" if cut in {"outcome", "admission"} else "ambiguous" + ) + if cut in {"outcome", "admission"}: + assert int(recovered["checkpoint"]["revision"]) == 3 + assert recovered["journal"]["effect_records"][0]["outcome"] == record["outcome"] + else: + assert recovered["checkpoint"] == before["checkpoint"] + assert recovered["journal"]["effect_records"][0]["outcome"] is None + saved = copy.deepcopy(recovered) + assert host.recover(information["root"]) == saved From 9b6b770b82ced5cf36812e0644f119786eb6af8d Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 21:57:09 +0000 Subject: [PATCH 22/32] Keep dispatch starts private and reject incomplete native inventories (WIP) --- src/determa/state/authority.py | 22 +++- src/determa/state/effects.py | 129 +++++++++++++++------- tests/test_committed_effects.py | 65 +++++++++-- tests/test_effect_dispatch_native_fate.py | 19 +--- tests/test_local_authority.py | 85 ++++++++++++++ 5 files changed, 257 insertions(+), 63 deletions(-) diff --git a/src/determa/state/authority.py b/src/determa/state/authority.py index 850daf7..cfe3675 100644 --- a/src/determa/state/authority.py +++ b/src/determa/state/authority.py @@ -313,18 +313,27 @@ def _native_checkpoints( "ORDER BY root_instance_id" ) ) + from .effects import _native_effect_documents, validate_journal + + native_documents = _native_effect_documents(ledger) + if native_documents and "determa_committed_effects" not in tables: + raise ValueError("native effect participant table missing") if "determa_committed_effects" in tables: - from .effects import validate_journal from .wire import canonical_bytes - for root, payload in connection.execute( + native_rows = connection.execute( "SELECT root_instance_id, document FROM determa_committed_effects " "ORDER BY root_instance_id" - ): + ).fetchall() + if {root for root, _payload in native_rows} != set(native_documents): + raise ValueError("native effect participant inventory differs") + for root, payload in native_rows: native = _parse(bytes(payload).decode("utf-8", "strict")) validate_journal(native["checkpoint"], native["journal"]) if native["journal"]["scope_identity"] != ledger["scope_identity"]: raise ValueError("untracked native effect scope") + if native != native_documents[root]: + raise ValueError("native effect private history differs") for record in native["journal"]["effect_records"]: if { "work_identity": record["effect_id"], @@ -1049,6 +1058,13 @@ def _perform( and invocation["authenticated_principal"] != ledger["owner_principal"] ): return _compact(_result(request, ledger, "stale_scope_authority")) + # A new mutation cannot certify a scope around a missing private + # native participant. Historical receipt replay above grants no rights. + if ledger.get("native_effect_roots") or ledger.get("native_effect_document_bytes"): + try: + _native_checkpoints(connection, ledger) + except (ValueError, TypeError, KeyError, sqlite3.Error): + return _compact(_result(request, ledger, "scope_fence_unproven")) new_claim = None if operation == "fence_worker": arguments = request["arguments"] diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index a73dd73..82d2199 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -716,6 +716,48 @@ def _native_effect_journal(ledger: Mapping[str, Any], root: str) -> dict[str, An return latest +def _native_effect_documents(ledger: Mapping[str, Any]) -> dict[str, dict[str, Any]]: + """Only independently retained native-role bytes certify private call history.""" + roots = ledger.get("native_effect_roots", []) + if ( + not isinstance(roots, list) + or any(type(root) is not str for root in roots) + or len(set(roots)) != len(roots) + ): + raise EffectError("unauthorized_scope") + history = ledger.get("native_effect_document_bytes", [] if not roots else None) + if not isinstance(history, list): + raise EffectError("unauthorized_scope") + latest = {} + for source in history: + if type(source) is not str: + raise EffectError("unauthorized_scope") + try: + value = json.loads(source) + if not isinstance(value, dict) or canonical_bytes(value).decode() != source: + raise ValueError("invalid native participant bytes") + if value["journal"]["scope_identity"] != ledger["scope_identity"]: + raise ValueError("wrong native scope") + if not isinstance(value["invocation_starts"], dict): + raise ValueError("missing private start inventory") + root = value["checkpoint"]["root_instance_id"] + if type(root) is not str or value["journal"]["root_instance_id"] != root: + raise ValueError("wrong native root") + latest[root] = value + except (ValueError, TypeError, KeyError) as error: + raise EffectError("unauthorized_scope") from error + if set(latest) != set(roots): + raise EffectError("unauthorized_scope") + return latest + + +def _native_effect_document(ledger: Mapping[str, Any], root: str) -> dict[str, Any]: + latest = _native_effect_documents(ledger).get(root) + if latest is None: + raise EffectError("unauthorized_scope") + return latest + + def _validate_authority_pair(ledger: Mapping[str, Any], document: Mapping[str, Any]) -> None: """Check the old native pair before any journal transition can be staged.""" from .authority import _native_checkpoint_history @@ -772,6 +814,8 @@ def _validate_authority_pair(ledger: Mapping[str, Any], document: Mapping[str, A raise EffectError("stale_attempt_fence") if journal != _native_effect_journal(ledger, journal["root_instance_id"]): raise EffectError("unauthorized_scope") + if document != _native_effect_document(ledger, journal["root_instance_id"]): + raise EffectError("unauthorized_scope") def _issue_effect_claim( @@ -985,6 +1029,7 @@ def seed( "result_responses": {}, "cancel_requests": {}, "producer_requests": {}, + "invocation_starts": {}, } if claim is not None: document["claims"][claim["work_identity"]] = copy.deepcopy(dict(claim)) @@ -1016,6 +1061,18 @@ def seed( "native_effect_roots", [] ) and journal != _native_effect_journal(ledger, checkpoint["root_instance_id"]): raise EffectError("unauthorized_scope") + if checkpoint["root_instance_id"] in ledger.get("native_effect_roots", []): + retained = _native_effect_document(ledger, checkpoint["root_instance_id"]) + # Portable equality never restores private call permission. + # Existing starts require explicit native recovery, not seed. + if retained["invocation_starts"] or retained["checkpoint"] != checkpoint: + raise EffectError("unauthorized_scope") + if ( + claim is not None + and retained["claims"].get(claim["work_identity"]) != claim + ): + raise EffectError("stale_attempt_fence") + document = copy.deepcopy(retained) # Restoring an already claimed invocation cannot mint authority. # Its exact claim must already have been issued by this ledger. if claim is not None and ( @@ -1067,6 +1124,10 @@ def seed( ): raise EffectError("stale_attempt_fence") self._mirror_authority(ledger, document) + connection.execute( + "UPDATE determa_committed_effects SET document=? WHERE root_instance_id=?", + (canonical_bytes(document), checkpoint["root_instance_id"]), + ) connection.execute( "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", ( @@ -1081,8 +1142,12 @@ def _mirror_authority( ledger: dict[str, Any], document: dict[str, Any], *, advance_generation: bool = True ) -> None: """Mirror effect fences and exact committed bytes in the colocated authority ledger.""" + _native_effect_documents(ledger) native_roots = ledger.setdefault("native_effect_roots", []) root = document["checkpoint"]["root_instance_id"] + fresh = root not in native_roots + if not fresh: + _native_effect_document(ledger, root) if root not in native_roots: native_roots.append(root) records = document["journal"]["effect_records"] @@ -1143,6 +1208,9 @@ def _mirror_authority( ledger.setdefault("native_effect_journal_bytes", []).append( canonical_bytes(document["journal"]).decode() ) + ledger.setdefault("native_effect_document_bytes", []).append( + canonical_bytes(document).decode() + ) if advance_generation: ledger["scope_generation"] = str(int(ledger["scope_generation"]) + 1) @@ -1212,6 +1280,12 @@ def _transact( ): raise EffectError("stale_scope_authority") _validate_authority_pair(ledger, document) + from .authority import _native_checkpoints + + try: + _native_checkpoints(connection, ledger) + except (ValueError, TypeError, KeyError, sqlite3.Error) as error: + raise EffectError("unauthorized_scope") from error if worker_guard is not None: effect_id, principal, fence, trusted_now = worker_guard journal_record = _record(document["journal"], effect_id) @@ -1630,27 +1704,20 @@ def start(document: dict[str, Any]) -> str: record["attempt_fence"], ] ) - journal = document["journal"] - if identifier in document["responses"] or any( - reference["operation_id"] == identifier - for reference in journal["operation_response_references"] - ): + starts = document["invocation_starts"] + if identifier in starts: raise EffectError("native_invocation_already_started") - response = { - "kind": "effect_invocation_start", - "body": {"claim": copy.deepcopy(document["claims"][effect_id])}, - } - document["responses"][identifier] = response - journal["operation_response_references"].append( - { - "operation_id": identifier, - "response_digest": hash_value(["determa-host-operation-response-1", response]), - } - ) - journal["operation_response_references"].sort( - key=lambda reference: reference["operation_id"] + intent = next( + item["intent"] + for item in document["checkpoint"]["pending_outbox_intents"] + + document["checkpoint"]["terminal_outbox_records"] + if item["intent"]["effect_id"] == effect_id ) - _bump(journal) + starts[identifier] = { + "claim": copy.deepcopy(document["claims"][effect_id]), + "intent": copy.deepcopy(intent), + "effect_record": copy.deepcopy(record), + } return identifier # Only this successful new native commit reaches the call phase. A saved @@ -1665,24 +1732,10 @@ def start(document: dict[str, Any]) -> str: def call(document: dict[str, Any]) -> Mapping[str, Any]: record = _record(document["journal"], effect_id) - response = document["responses"].get(start_id) - reference = next( - ( - item - for item in document["journal"]["operation_response_references"] - if item["operation_id"] == start_id - ), - None, - ) - if ( - response - != { - "kind": "effect_invocation_start", - "body": {"claim": document["claims"].get(effect_id)}, - } - or reference is None - or reference["response_digest"] - != hash_value(["determa-host-operation-response-1", response]) + retained_start = document["invocation_starts"].get(start_id) + if not isinstance(retained_start, dict) or ( + retained_start.get("claim") != document["claims"].get(effect_id) + or retained_start.get("effect_record") != record ): raise EffectError("invalid_effect_journal") if record["invocation_state"] != "leased": @@ -1712,6 +1765,8 @@ def call(document: dict[str, Any]) -> Mapping[str, Any]: ) if item["intent"]["effect_id"] == effect_id ) + if retained_start.get("intent") != intent: + raise EffectError("invalid_effect_journal") metadata = { "scope_identity": scope, "effect_id": effect_id, diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index cb3cacc..9d2d7b3 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -976,10 +976,9 @@ def external_acceptance(*args): after = host.snapshot(root) assert after["checkpoint"] == before["checkpoint"] assert after["journal"]["effect_records"] == before["journal"]["effect_records"] - assert ( - int(after["journal"]["journal_revision"]) == int(before["journal"]["journal_revision"]) + 1 - ) - assert len(after["responses"]) == len(before["responses"]) + 1 + assert after["journal"] == before["journal"] + assert after["responses"] == before["responses"] + assert len(after["invocation_starts"]) == 1 current_time[0] = "0" with pytest.raises(EffectError, match="native_invocation_already_started"): host.dispatch(root, request["effect_id"], credential="test-credential", **context) @@ -1847,10 +1846,7 @@ def health(self, instance): document = json.loads( connection.execute("SELECT document FROM determa_committed_effects").fetchone()[0] ) - if any( - response.get("kind") == "effect_invocation_start" - for response in document["responses"].values() - ): + if document["invocation_starts"]: current_time[0] = expiry return "healthy" @@ -1863,7 +1859,7 @@ def health(self, instance): after = host.snapshot(root) assert after["checkpoint"] == before["checkpoint"] assert after["journal"]["effect_records"] == before["journal"]["effect_records"] - assert len(after["responses"]) == 1 + assert len(after["invocation_starts"]) == 1 assert calls == [] @@ -2013,3 +2009,54 @@ def test_native_authority_seed_cannot_erase_a_committed_dispatch_start(tmp_path) host.seed(before["checkpoint"], before["journal"], claim) assert authority.inspect(scope) == ledger assert len(calls) == 1 + + +@pytest.mark.parametrize("damage", ["erase_start", "older_document", "missing_native_history"]) +def test_private_dispatch_start_loss_cannot_be_blessed_by_dispatch_or_authority(tmp_path, damage): + import sqlite3 + + authority, host, scope, root, record = authority_effect_fixture(tmp_path) + command, invocation = authority_claim_request(authority, scope, root, record) + claim = json.loads(authority.perform(command, invocation))["claim"] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + host.trusted_clock = lambda: "0" + calls = [] + host = installed_test_handler(host, lambda *args: calls.append(args) or {"accepted": True}) + context = { + "principal": claim["worker_principal"], + "scope": scope, + "epoch": "0", + "trusted_now": "0", + } + before = host.snapshot(root) + host.dispatch(root, record["effect_id"], credential="test-credential", **context) + after = host.snapshot(root) + assert after["checkpoint"] == before["checkpoint"] + assert after["journal"] == before["journal"] + assert after["responses"] == before["responses"] + assert len(after["invocation_starts"]) == 1 + fresh_command, fresh_invocation = authority_claim_request( + authority, scope, root, record, operation="after-private-start", expected="1" + ) + with sqlite3.connect(host.path) as connection: + if damage == "missing_native_history": + ledger = json.loads( + connection.execute("SELECT ledger FROM determa_scope_authority").fetchone()[0] + ) + del ledger["native_effect_document_bytes"] + connection.execute("UPDATE determa_scope_authority SET ledger=?", (json.dumps(ledger),)) + else: + corrupted = copy.deepcopy(after if damage == "erase_start" else before) + corrupted["invocation_starts"] = {} + connection.execute( + "UPDATE determa_committed_effects SET document=?", (json.dumps(corrupted).encode(),) + ) + with pytest.raises(EffectError, match="unauthorized_scope"): + host.dispatch(root, record["effect_id"], credential="test-credential", **context) + # The authority's generic mutation validation must not certify the corrupt + # private inventory by mirroring it into a new trusted native history entry. + assert json.loads(authority.perform(command, invocation))["claim"] == claim + assert json.loads(authority.perform(fresh_command, fresh_invocation))["status"] == "rejected" + assert len(calls) == 1 diff --git a/tests/test_effect_dispatch_native_fate.py b/tests/test_effect_dispatch_native_fate.py index dfd0c05..9f03846 100644 --- a/tests/test_effect_dispatch_native_fate.py +++ b/tests/test_effect_dispatch_native_fate.py @@ -50,10 +50,7 @@ def health(self, instance): 0 ] ) - if any( - response.get("kind") == "effect_invocation_start" - for response in document["responses"].values() - ): + if document["invocation_starts"]: control.send(information) time.sleep(60) return "healthy" @@ -176,16 +173,10 @@ def test_sigkill_native_dispatch_never_repeats_uncertain_provider_call(tmp_path, ) assert host.snapshot(information["root"]) == before assert calls == [] - assert ( - len( - [ - reply - for reply in before["responses"].values() - if reply.get("kind") == "effect_invocation_start" - ] - ) - == 1 - ) + assert len(before["invocation_starts"]) == 1 + start = next(iter(before["invocation_starts"].values())) + assert start["claim"] == information["claim"] + assert start["effect_record"]["effect_id"] == information["effect_id"] assert record["invocation_state"] == ( "result_admitted" if cut == "admission" diff --git a/tests/test_local_authority.py b/tests/test_local_authority.py index 7d49f42..f8b5c88 100644 --- a/tests/test_local_authority.py +++ b/tests/test_local_authority.py @@ -274,6 +274,91 @@ def test_freeze_refuses_an_untracked_native_checkpoint_without_committing(tmp_pa assert bare_host.read_checkpoint("foreign-root").canonical_bytes == before.canonical_bytes +@pytest.mark.parametrize("damage", [None, "row", "table", "history", "ownership"]) +def test_freeze_requires_complete_private_native_inventory_even_for_zero_effect_roots( + tmp_path, damage +): + from determa.state import create_checkpoint_v1 + from determa.state.effects import SQLiteCommittedEffectHost, seal_journal + + bundle = load_bundle( + "format: 1\nnamespace: test.private_inventory\nmachines:\n" + " - machine_id: simple\n root: {type: simple}\n" + ) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + authority = SQLiteLocalAuthority(tmp_path / "inventory.sqlite") + authority.setup_schema() + assert authority.allocate("scope-1", "owner-1", roots=("root-a", "root-b")) + host = SQLiteCommittedEffectHost( + authority.path, resolver, {"authority_epoch": "0"}, None, authority_scope="scope-1" + ) + host.setup_schema() + for root in ("root-a", "root-b"): + checkpoint = create_checkpoint_v1(bundle, "simple", root, "create-" + root, {}) + document = checkpoint + journal = seal_journal( + { + "host_effect_journal_format": "determa.host_effect_journal", + "host_effect_journal_schema_version": 1, + "scope_identity": "scope-1", + "root_instance_id": root, + "checkpoint_revision": document["revision"], + "checkpoint_digest": document["execution_checkpoint_digest"], + "journal_revision": "0", + "effect_records": [], + "operation_response_references": [], + } + ) + host.seed(document, journal) + before = authority.inspect("scope-1") + if damage is not None: + with authority._connect() as connection: + if damage == "row": + connection.execute( + "DELETE FROM determa_committed_effects WHERE root_instance_id='root-b'" + ) + elif damage == "table": + connection.execute("DROP TABLE determa_committed_effects") + else: + ledger = json.loads( + connection.execute("SELECT ledger FROM determa_scope_authority").fetchone()[0] + ) + if damage == "history": + ledger["native_effect_document_bytes"] = [ + source + for source in ledger["native_effect_document_bytes"] + if json.loads(source)["checkpoint"]["root_instance_id"] != "root-b" + ] + else: + ledger["native_effect_roots"].remove("root-b") + connection.execute( + "UPDATE determa_scope_authority SET ledger=?", (json.dumps(ledger),) + ) + invocation = { + "authenticated_principal": "owner-1", + "authorized_scopes": ["scope-1"], + "operation_rights": ["freeze_scope"], + } + result = json.loads( + authority.perform( + _request("freeze_scope", "freeze-native-inventory", before["scope_generation"], {}), + invocation, + ) + ) + if damage is None: + assert result["status"] == "accepted" and result["state"] == "frozen" + inventory = authority.inspect("scope-1")["inventory"] + assert {item["identity"] for item in inventory if item["kind"] == "root"} == { + "root-a", + "root-b", + } + assert sum(item["kind"] == "checkpoint" for item in inventory) >= 2 + else: + assert result["error_code"] == "scope_fence_unproven" + assert authority.inspect("scope-1")["state"] == "active" + assert authority.inspect("scope-1")["scope_generation"] == before["scope_generation"] + + def test_public_checkpoint_commit_serializes_with_freeze(tmp_path) -> None: bundle = load_bundle( """format: 1 From fdb4152e85b66c0a7db303c3fa7e25ecbb2dfd42 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 22:17:19 +0000 Subject: [PATCH 23/32] Refuse native reseeding when retained ownership roles disagree --- src/determa/state/authority.py | 10 +++++- src/determa/state/effects.py | 62 +++++++++++++++++++++++++++------ tests/test_committed_effects.py | 31 ++++++++++++++--- tests/test_local_authority.py | 51 +++++++++++++++++++++++++-- 4 files changed, 136 insertions(+), 18 deletions(-) diff --git a/src/determa/state/authority.py b/src/determa/state/authority.py index cfe3675..d3ed6b7 100644 --- a/src/determa/state/authority.py +++ b/src/determa/state/authority.py @@ -1060,7 +1060,15 @@ def _perform( return _compact(_result(request, ledger, "stale_scope_authority")) # A new mutation cannot certify a scope around a missing private # native participant. Historical receipt replay above grants no rights. - if ledger.get("native_effect_roots") or ledger.get("native_effect_document_bytes"): + if any( + key in ledger + for key in ( + "native_effect_roots", + "native_effect_document_bytes", + "native_effect_journal_bytes", + "native_effect_work", + ) + ): try: _native_checkpoints(connection, ledger) except (ValueError, TypeError, KeyError, sqlite3.Error): diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 82d2199..4922171 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -686,31 +686,38 @@ def visit(value: Any, pointer: str) -> None: return response -def _native_effect_journal(ledger: Mapping[str, Any], root: str) -> dict[str, Any]: +def _native_effect_journals(ledger: Mapping[str, Any]) -> dict[str, dict[str, Any]]: """Resolve only host-participant history, never generic proposed mutation bytes.""" from .authority import _parse - history = ledger.get("native_effect_journal_bytes") + history = ledger.get("native_effect_journal_bytes", []) if type(history) is not list or any(type(source) is not str for source in history): raise EffectError("unauthorized_scope") - latest = None + latest = {} for source in history: try: journal = _parse(source) if type(journal) is not dict: raise EffectError("unauthorized_scope") + root = journal["root_instance_id"] if ( canonical_bytes(journal).decode() != source or journal.get("host_effect_journal_format") != "determa.host_effect_journal" or journal.get("host_effect_journal_schema_version") != 1 or journal.get("scope_identity") != ledger["scope_identity"] or journal.get("host_effect_journal_digest") != journal_digest(journal) + or type(root) is not str + or not root ): raise EffectError("unauthorized_scope") + latest[root] = journal except (ValueError, TypeError, KeyError) as error: raise EffectError("unauthorized_scope") from error - if journal["root_instance_id"] == root: - latest = journal + return latest + + +def _native_effect_journal(ledger: Mapping[str, Any], root: str) -> dict[str, Any]: + latest = _native_effect_journals(ledger).get(root) if latest is None: raise EffectError("unauthorized_scope") return latest @@ -721,10 +728,23 @@ def _native_effect_documents(ledger: Mapping[str, Any]) -> dict[str, dict[str, A roots = ledger.get("native_effect_roots", []) if ( not isinstance(roots, list) - or any(type(root) is not str for root in roots) + or any(type(root) is not str or not root for root in roots) or len(set(roots)) != len(roots) ): raise EffectError("unauthorized_scope") + journals = _native_effect_journals(ledger) + # Losing both indexes cannot turn retained native journal/work into freshness. + if set(journals) != set(roots): + raise EffectError("unauthorized_scope") + work = ledger.get("native_effect_work", []) + if type(work) is not list or any( + type(item) is not dict + or item.get("participant") != "native_effects" + or item.get("work_kind") != "effect" + or item.get("root_instance_id") not in roots + for item in work + ): + raise EffectError("unauthorized_scope") history = ledger.get("native_effect_document_bytes", [] if not roots else None) if not isinstance(history, list): raise EffectError("unauthorized_scope") @@ -736,17 +756,38 @@ def _native_effect_documents(ledger: Mapping[str, Any]) -> dict[str, dict[str, A value = json.loads(source) if not isinstance(value, dict) or canonical_bytes(value).decode() != source: raise ValueError("invalid native participant bytes") - if value["journal"]["scope_identity"] != ledger["scope_identity"]: - raise ValueError("wrong native scope") + if ( + value["journal"]["scope_identity"] != ledger["scope_identity"] + or value["journal"]["host_effect_journal_format"] != "determa.host_effect_journal" + or value["journal"]["host_effect_journal_schema_version"] != 1 + or value["journal"]["host_effect_journal_digest"] + != journal_digest(value["journal"]) + ): + raise ValueError("wrong native journal provenance") if not isinstance(value["invocation_starts"], dict): raise ValueError("missing private start inventory") root = value["checkpoint"]["root_instance_id"] - if type(root) is not str or value["journal"]["root_instance_id"] != root: + if type(root) is not str or not root or value["journal"]["root_instance_id"] != root: raise ValueError("wrong native root") latest[root] = value except (ValueError, TypeError, KeyError) as error: raise EffectError("unauthorized_scope") from error - if set(latest) != set(roots): + if set(latest) != set(roots) or any( + document["journal"] != journals[root] for root, document in latest.items() + ): + raise EffectError("unauthorized_scope") + expected_work = [ + { + "root_instance_id": root, + "work_kind": "effect", + "work_identity": record["effect_id"], + "operation_token": record["operation_token"], + "participant": "native_effects", + } + for root, document in latest.items() + for record in document["journal"]["effect_records"] + ] + if len(work) != len(expected_work) or any(item not in work for item in expected_work): raise EffectError("unauthorized_scope") return latest @@ -1057,6 +1098,7 @@ def seed( or journal["scope_identity"] != self.authority_scope ): raise EffectError("unauthorized_scope") + _native_effect_documents(ledger) if checkpoint["root_instance_id"] in ledger.get( "native_effect_roots", [] ) and journal != _native_effect_journal(ledger, checkpoint["root_instance_id"]): diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 9d2d7b3..58401b2 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -1979,7 +1979,8 @@ def native_call(*args): assert other.submit_result(root, request, **context)["status"] == "committed" -def test_native_authority_seed_cannot_erase_a_committed_dispatch_start(tmp_path): +@pytest.mark.parametrize("damage", [None, "missing_indexes", "empty_indexes"]) +def test_native_authority_seed_cannot_erase_a_committed_dispatch_start(tmp_path, damage): import sqlite3 authority, host, scope, root, record = authority_effect_fixture(tmp_path) @@ -2000,18 +2001,34 @@ def test_native_authority_seed_cannot_erase_a_committed_dispatch_start(tmp_path) "trusted_now": "0", } host.dispatch(root, record["effect_id"], credential="test-credential", **context) - ledger = authority.inspect(scope) with sqlite3.connect(host.path) as connection: + if damage is not None: + retained = json.loads( + connection.execute("SELECT ledger FROM determa_scope_authority").fetchone()[0] + ) + for key in ("native_effect_roots", "native_effect_document_bytes"): + if damage == "missing_indexes": + del retained[key] + else: + retained[key] = [] + # Dedicated journal/work/issued-claim evidence still proves prior ownership. + assert retained["native_effect_journal_bytes"] and retained["native_effect_work"] + connection.execute( + "UPDATE determa_scope_authority SET ledger=?", (json.dumps(retained),) + ) connection.execute( "DELETE FROM determa_committed_effects WHERE root_instance_id=?", (root,) ) + ledger = authority.inspect(scope) with pytest.raises(EffectError, match="unauthorized_scope"): host.seed(before["checkpoint"], before["journal"], claim) assert authority.inspect(scope) == ledger assert len(calls) == 1 -@pytest.mark.parametrize("damage", ["erase_start", "older_document", "missing_native_history"]) +@pytest.mark.parametrize( + "damage", ["erase_start", "older_document", "missing_native_history", "missing_native_work"] +) def test_private_dispatch_start_loss_cannot_be_blessed_by_dispatch_or_authority(tmp_path, damage): import sqlite3 @@ -2041,11 +2058,15 @@ def test_private_dispatch_start_loss_cannot_be_blessed_by_dispatch_or_authority( authority, scope, root, record, operation="after-private-start", expected="1" ) with sqlite3.connect(host.path) as connection: - if damage == "missing_native_history": + if damage in {"missing_native_history", "missing_native_work"}: ledger = json.loads( connection.execute("SELECT ledger FROM determa_scope_authority").fetchone()[0] ) - del ledger["native_effect_document_bytes"] + del ledger[ + "native_effect_document_bytes" + if damage == "missing_native_history" + else "native_effect_work" + ] connection.execute("UPDATE determa_scope_authority SET ledger=?", (json.dumps(ledger),)) else: corrupted = copy.deepcopy(after if damage == "erase_start" else before) diff --git a/tests/test_local_authority.py b/tests/test_local_authority.py index f8b5c88..857b802 100644 --- a/tests/test_local_authority.py +++ b/tests/test_local_authority.py @@ -274,7 +274,23 @@ def test_freeze_refuses_an_untracked_native_checkpoint_without_committing(tmp_pa assert bare_host.read_checkpoint("foreign-root").canonical_bytes == before.canonical_bytes -@pytest.mark.parametrize("damage", [None, "row", "table", "history", "ownership"]) +@pytest.mark.parametrize( + "damage", + [ + None, + "row", + "table", + "history", + "ownership", + "missing_indexes", + "empty_indexes", + "journal_history", + "missing_journal_history", + "wrong_journal_scope", + "unowned_work", + "malformed_work", + ], +) def test_freeze_requires_complete_private_native_inventory_even_for_zero_effect_roots( tmp_path, damage ): @@ -329,8 +345,39 @@ def test_freeze_requires_complete_private_native_inventory_even_for_zero_effect_ for source in ledger["native_effect_document_bytes"] if json.loads(source)["checkpoint"]["root_instance_id"] != "root-b" ] - else: + elif damage == "ownership": ledger["native_effect_roots"].remove("root-b") + elif damage in {"missing_indexes", "empty_indexes"}: + for key in ("native_effect_roots", "native_effect_document_bytes"): + if damage == "missing_indexes": + del ledger[key] + else: + ledger[key] = [] + connection.execute("DELETE FROM determa_committed_effects") + elif damage == "missing_journal_history": + del ledger["native_effect_journal_bytes"] + elif damage in {"journal_history", "wrong_journal_scope"}: + latest = json.loads(ledger["native_effect_journal_bytes"][-1]) + if damage == "journal_history": + latest["journal_revision"] = "99" + else: + latest["scope_identity"] = "other-scope" + from determa.state.wire import canonical_bytes + + ledger["native_effect_journal_bytes"][-1] = canonical_bytes( + seal_journal(latest) + ).decode() + elif damage == "unowned_work": + ledger["native_effect_work"] = [ + { + "participant": "native_effects", + "work_kind": "effect", + "root_instance_id": "unowned-root", + } + ] + else: + assert damage == "malformed_work" + ledger["native_effect_work"] = None connection.execute( "UPDATE determa_scope_authority SET ledger=?", (json.dumps(ledger),) ) From 4d08e393b405d0680cbb259466e4ef967e51758b Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Tue, 6 Oct 2026 06:49:01 +0000 Subject: [PATCH 24/32] fix(effects): reverify destination retry proof after native staging --- src/determa/state/effects.py | 26 +++++++++- tests/test_committed_effects.py | 88 +++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 4922171..27f2d4c 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -1277,6 +1277,7 @@ def _transact( expected_epoch: str | None = None, authority_mutation: Callable[[dict[str, Any], dict[str, Any]], None] | None = None, worker_guard: tuple[str, str, str | None, str] | None = None, + before_native_commit: Callable[[], None] | None = None, ) -> _T: with self._connect() as connection: connection.execute("BEGIN IMMEDIATE") @@ -1385,6 +1386,8 @@ def _transact( "UPDATE determa_committed_effects SET document = ? WHERE root_instance_id = ?", (canonical_bytes(document), root), ) + if before_native_commit is not None: + before_native_commit() if expiry_guard is not None: # Keep the original deadline even when this transaction closes # or revokes the claim. Worker rights must still hold at commit. @@ -1548,7 +1551,19 @@ def claim( trusted_now: str, deduplication_evidence: Mapping[str, Any] | None = None, ) -> dict[str, Any]: + final_proof: tuple[dict[str, Any], str, dict[str, Any]] | None = None + + def verify_before_commit() -> None: + if final_proof is None: + return + reference, destination, proof = final_proof + handler = self._verified_handler(reference, destination) + VerifiedNativeHandler.verify_deduplication_evidence( + handler, reference, destination, proof + ) + def change(document: dict[str, Any]) -> dict[str, Any]: + nonlocal final_proof record = _record(document["journal"], effect_id) proof = None if deduplication_evidence is not None: @@ -1589,6 +1604,11 @@ def change(document: dict[str, Any]) -> dict[str, Any]: record["destination_binding_digest"], proof, ) + final_proof = ( + copy.deepcopy(record["handler_reference"]), + record["destination_binding_digest"], + copy.deepcopy(proof), + ) claim = _issue_effect_claim( document, effect_id, @@ -1611,7 +1631,11 @@ def change(document: dict[str, Any]) -> dict[str, Any]: return claim return self._transact( - root, change, expected_epoch=epoch, authority_mutation=self._mirror_authority + root, + change, + expected_epoch=epoch, + authority_mutation=self._mirror_authority, + before_native_commit=verify_before_commit, ) def terminalize_outbox( diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 58401b2..de216ef 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -1430,6 +1430,94 @@ def test_ambiguous_retry_does_not_accept_legacy_boolean_keyword(tmp_path): ) +@pytest.mark.parametrize("damage", [None, "unavailable", "method", "configuration", "instance"]) +def test_retry_verifier_is_rechecked_after_native_sql_staging(tmp_path, monkeypatch, damage): + import base64 + import sqlite3 + + original, root, request, _ = host_fixture(tmp_path) + journal = read("data/ambiguous-journal.json") + host = SQLiteCommittedEffectHost(tmp_path / "retry-staging.sqlite", original.resolver, {}, None) + host.setup_schema() + host.seed(read("pending-checkpoint.json"), journal) + record = journal["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + receipt = b"independently stored destination receipt" + native_path = tmp_path / "retry-destination.sqlite" + with sqlite3.connect(native_path) as connection: + connection.execute("CREATE TABLE receipts (receipt BLOB NOT NULL)") + connection.execute("INSERT INTO receipts VALUES (?)", (receipt,)) + state = {"staged": False, "checks": 0} + + def verify_receipts(evidence): + state["checks"] += 1 + if state["staged"] and damage == "unavailable": + raise RuntimeError("native verifier unavailable after staging") + with sqlite3.connect(native_path) as connection: + stored = connection.execute("SELECT receipt FROM receipts").fetchone()[0] + return all( + base64.b64decode(evidence[name], validate=True) == stored + for name in ( + "first_attempt_receipt_bytes_base64", + "repeat_attempt_receipt_bytes_base64", + ) + ) + + host = installed_test_handler(host, lambda *_: {}, verify_receipts) + encoded = base64.b64encode(receipt).decode() + proof = { + "scope_identity": journal["scope_identity"], + "effect_id": record["effect_id"], + "destination_binding_digest": record["destination_binding_digest"], + "first_attempt_receipt_bytes_base64": encoded, + "repeat_attempt_receipt_bytes_base64": encoded, + } + before = host.snapshot(root) + + class StagingConnection(sqlite3.Connection): + def execute(self, sql, parameters=()): + cursor = super().execute(sql, parameters) + if sql.startswith("UPDATE determa_committed_effects SET document"): + # Observe the actual uncommitted row after SQLite has written it. + staged = json.loads( + super() + .execute( + "SELECT document FROM determa_committed_effects WHERE root_instance_id=?", + (root,), + ) + .fetchone()[0] + ) + assert staged["claims"][request["effect_id"]]["attempt_fence"] == "2" + state["staged"] = True + if damage == "method": + monkeypatch.setattr( + NativeTestProvider, "verify_deduplication_evidence", lambda *_: True + ) + elif damage == "configuration": + host.handler._configured._configuration["instance_id"] = "substituted" + elif damage == "instance": + object.__setattr__(host.handler._configured, "_instance", {}) + return cursor + + monkeypatch.setattr( + host, + "_connect", + lambda: sqlite3.connect(host.path, isolation_level=None, factory=StagingConnection), + ) + arguments = {"expires_at": "10", "trusted_now": "0", "deduplication_evidence": proof} + if damage is None: + claim = host.claim(root, request["effect_id"], "worker", "0", **arguments) + assert claim["attempt_fence"] == "2" + assert state["staged"] and state["checks"] >= 2 + else: + with pytest.raises(EffectError, match="host_capability_mismatch"): + host.claim(root, request["effect_id"], "worker", "0", **arguments) + assert state["staged"] + assert host.snapshot(root) == before + + def test_historical_admission_survives_actual_root_completion_and_tombstone(tmp_path): import yaml From 8b5fdf75c2d40f6691df7c316295fa6fe3593fff Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Tue, 6 Oct 2026 07:03:05 +0000 Subject: [PATCH 25/32] feat(effects): verify scoped native retry decisions (WIP) --- AGENTS.md | 5 +- conformance/effects_adapter.py | 82 ++++++-- conformance/pins.py | 2 +- src/determa/state/effects.py | 296 +++++++++++++++++++++------ tests/test_committed_effects.py | 345 ++++++++++++++++++++++++++++++-- 5 files changed, 632 insertions(+), 98 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4b87733..d68d6af 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,10 +12,11 @@ The implementation is conformant only when it passes the language-neutral suite. The implementation uses these immutable inputs: - specification: `77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0`; -- conformance: `dc84ed81ea36a5f2140181a97660477a14347ccc` (99 format-1 +- conformance: `384103d3709819052b3daaae6f89b531c5c3c146` (reviewed coordinated + draft #114; 99 format-1 core cases, 162 version-1 vectors, 142 durable-host vectors, 49 core inspection vectors, 47 extension negotiation vectors, 7 optional native-provider vectors, - 52 runtime-provider vectors, and 487 JSON artifacts). + 52 runtime-provider vectors, 55 committed-native-effect vectors, and 487 JSON artifacts). The package metadata is `0.3.0`. diff --git a/conformance/effects_adapter.py b/conformance/effects_adapter.py index 9b59466..12527f1 100644 --- a/conformance/effects_adapter.py +++ b/conformance/effects_adapter.py @@ -396,6 +396,7 @@ def equal(left: Any, right: Any) -> bool: class _FixtureNativeProvider: def __init__(self, invocation: Any) -> None: self.invocation = invocation + self.retry_checker_available = True def validate_configuration(self, configuration: dict[str, Any]) -> dict[str, Any]: return copy.deepcopy(configuration) @@ -410,9 +411,21 @@ def invoke(self, instance: Any, payload: Any, metadata: Any, attempt: Any) -> An return self.invocation(payload, metadata, attempt) def verify_deduplication_evidence(self, instance: Any, evidence: Any) -> bool: - return instance["destination_binding_digest"] == _sha( - _destination_configuration() - ) and evidence == _destination_proof(evidence["scope_identity"], evidence["effect_id"]) + return ( + self.retry_checker_available + and instance["destination_binding_digest"] == _sha(_destination_configuration()) + and { + key: evidence[key] + for key in ( + "scope_identity", + "effect_id", + "destination_binding_digest", + "first_attempt_receipt_bytes_base64", + "repeat_attempt_receipt_bytes_base64", + ) + } + == _destination_proof(evidence["scope_identity"], evidence["effect_id"]) + ) def _run_operation(payload: dict[str, Any], control: Path | None = None) -> dict[str, Any]: @@ -581,6 +594,43 @@ def verify_native(source: Any, selected: Any) -> bool: fault = payload["fault"] response: dict[str, Any] | None = None caller_kind = "completed" + retry_evidence = None + original_record = next( + ( + record + for record in journal["effect_records"] + if record["effect_id"] == arguments.get("effect_id") + ), + None, + ) + retry_decision = ( + operation == "submit_result" and arguments["outcome_kind"] == "retryable_failure" + ) or ( + operation == "claim" + and original_record is not None + and original_record["attempt_fence"] != "0" + ) + if retry_decision and configuration["destination_deduplication_proven"]: + _destination_call(_AUTHORITY_SCOPE, arguments["effect_id"], None) + _destination_call(_AUTHORITY_SCOPE, arguments["effect_id"], None) + retry_evidence = { + **SQLiteCommittedEffectHost._retry_context(before, original_record), + **_destination_proof(_AUTHORITY_SCOPE, arguments["effect_id"]), + } + if fault == "retry_safety_missing": + retry_evidence = None + elif fault == "retry_safety_boolean": + retry_evidence = True + elif fault == "retry_safety_fabricated": + retry_evidence["first_attempt_receipt_bytes_base64"] = retry_evidence[ + "repeat_attempt_receipt_bytes_base64" + ] = base64.b64encode(b"fabricated equal receipts").decode() + elif fault == "retry_safety_wrong_work": + retry_evidence["effect_id"] = "sha256:" + "0" * 64 + elif fault == "retry_safety_wrong_destination": + retry_evidence["destination_binding_digest"] = "sha256:" + "0" * 64 + elif fault == "retry_safety_verifier_unavailable": + provider.retry_checker_available = False if operation in {"produce", "produce_replay"}: request = arguments["original_request"] aggregate = checkpoint["root_record"]["aggregate_state"] @@ -641,17 +691,6 @@ def before_commit(candidate: dict[str, Any], proposed_journal: dict[str, Any]) - caller_kind = "aborted" elif operation == "claim": try: - evidence = None - if configuration["destination_deduplication_proven"] and any( - record["effect_id"] == arguments["effect_id"] - and record["invocation_state"] == "ambiguous" - for record in journal["effect_records"] - ): - # Exercise the actual installed destination twice. The configuration - # requests this probe; only native receipt verification permits retry. - _destination_call(_AUTHORITY_SCOPE, arguments["effect_id"], None) - _destination_call(_AUTHORITY_SCOPE, arguments["effect_id"], None) - evidence = _destination_proof(_AUTHORITY_SCOPE, arguments["effect_id"]) claim = host.claim( _ROOT, arguments["effect_id"], @@ -659,7 +698,7 @@ def before_commit(candidate: dict[str, Any], proposed_journal: dict[str, Any]) - context["scope_authority_epoch"], expires_at=str(int(context["trusted_host_now"]) + 1), trusted_now=context["trusted_host_now"], - deduplication_evidence=evidence, + deduplication_evidence=retry_evidence, ) new_claims.append( { @@ -704,6 +743,7 @@ def before_commit(candidate: dict[str, Any], proposed_journal: dict[str, Any]) - scope=context["scope_identity"], epoch=context["scope_authority_epoch"], trusted_now=context["trusted_host_now"], + deduplication_evidence=retry_evidence, ) caller_kind = "response" if fault == "after_outcome_before_admission": @@ -800,6 +840,18 @@ def before_commit(candidate: dict[str, Any], proposed_journal: dict[str, Any]) - "claim_guard_observed": operation in {"claim", "dispatch", "submit_result"}, "native_transaction_id": str(uuid.uuid4()), "destination_call_evidence": destination_call_evidence, + "retry_safety_evidence": ( + copy.deepcopy(after["destination_evidence"][arguments["effect_id"]][-1]["evidence"]) + if retry_decision + and ( + operation == "claim" + and new_claims + or operation == "submit_result" + and response is not None + and response.get("status") == "report_recorded" + ) + else None + ), }, } diff --git a/conformance/pins.py b/conformance/pins.py index fac42a1..9058ba6 100644 --- a/conformance/pins.py +++ b/conformance/pins.py @@ -4,7 +4,7 @@ from pathlib import Path -CONFORMANCE_COMMIT = "dc84ed81ea36a5f2140181a97660477a14347ccc" +CONFORMANCE_COMMIT = "384103d3709819052b3daaae6f89b531c5c3c146" SPEC_COMMIT = "77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0" ROOT = Path(__file__).resolve().parent.parent diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 27f2d4c..cdfa8bb 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -82,12 +82,21 @@ def _journal_validator() -> Any: def _validate_request(request: Mapping[str, Any], name: str) -> dict[str, Any]: from jsonschema import Draft202012Validator + from referencing import Resource if not isinstance(request, Mapping): raise EffectError("invalid_host_request") normalized = copy.deepcopy(dict(request)) source = json.loads((Path(__file__).parent / "data" / name).read_text()) - if not Draft202012Validator(source, registry=_schema_registry()).is_valid(normalized): + registry = _schema_registry() + for filename in ( + "host-effect-journal-v1.schema.json", + "provider-reference-v1.schema.json", + name, + ): + schema = json.loads((Path(__file__).parent / "data" / filename).read_text()) + registry = registry.with_resource(schema["$id"], Resource.from_contents(schema)) + if not Draft202012Validator(source, registry=registry).is_valid(normalized): raise EffectError("invalid_host_request") return normalized @@ -518,7 +527,7 @@ def _rejected_result(effect_id: str, fence: str, code: str) -> dict[str, Any]: def _report_reason(request: Mapping[str, Any]) -> str | None: kind = request["outcome_kind"] if kind == "retryable_failure": - return "no_call_proven" + return "destination_deduplication_proven" if kind == "ambiguous": return "provider_acceptance_unknown" return None @@ -529,22 +538,24 @@ def _retained_report( ) -> dict[str, Any] | None: """Compare complete immutable report evidence even after a process restart.""" fence, kind = request["attempt_fence"], request["outcome_kind"] - digest = hash_value( - [ - "determa-effect-attempt-report-1", - record["effect_id"], - record["operation_token"], - fence, - kind, - request["payload"], - _report_reason(request), - ] - ) return next( ( item for item in record["attempt_records"] - if item["attempt_fence"] == fence and item["report_digest"] == digest + if item["attempt_fence"] == fence + and item["report_kind"] == kind + and item["report_digest"] + == hash_value( + [ + "determa-effect-attempt-report-1", + record["effect_id"], + record["operation_token"], + fence, + kind, + request["payload"], + item["reason"], + ] + ) ), None, ) @@ -876,7 +887,7 @@ def _issue_effect_claim( record = _record(journal, effect_id) if record["invocation_state"] not in {"unclaimed", "ambiguous"}: raise EffectError("effect_not_outstanding") - if record["invocation_state"] == "ambiguous" and not ( + if record["attempt_fence"] != "0" and not ( deduplication_proven and record["idempotency_policy"] == "destination_deduplicates" ): raise EffectError("effect_not_outstanding") @@ -1040,6 +1051,54 @@ def _verified_handler( VerifiedNativeHandler.verify(self.handler, reference, destination) return self.handler + def _verify_retry_evidence( + self, context: Mapping[str, Any], evidence: Mapping[str, Any] | None + ) -> dict[str, Any]: + """Bind a native receipt decision to the original committed invocation.""" + receipt_fields = { + "first_attempt_receipt_bytes_base64", + "repeat_attempt_receipt_bytes_base64", + } + if not isinstance(evidence, Mapping) or set(evidence) != set(context) | receipt_fields: + raise EffectError("host_capability_mismatch") + proof = copy.deepcopy(dict(evidence)) + if any(proof[key] != value for key, value in context.items()): + raise EffectError("host_capability_mismatch") + try: + first = base64.b64decode(proof["first_attempt_receipt_bytes_base64"], validate=True) + repeated = base64.b64decode(proof["repeat_attempt_receipt_bytes_base64"], validate=True) + if not first or first != repeated: + raise ValueError("destination receipts differ") + except (ValueError, TypeError) as exc: + raise EffectError("host_capability_mismatch") from exc + handler = SQLiteCommittedEffectHost._verified_handler( + self, context["handler_reference"], context["destination_binding_digest"] + ) + VerifiedNativeHandler.verify_deduplication_evidence( + handler, context["handler_reference"], context["destination_binding_digest"], proof + ) + return proof + + @staticmethod + def _retry_context(document: Mapping[str, Any], record: Mapping[str, Any]) -> dict[str, Any]: + if record["idempotency_policy"] != "destination_deduplicates": + raise EffectError("host_capability_mismatch") + return { + "kind": "destination_deduplication", + "scope_identity": document["journal"]["scope_identity"], + "root_instance_id": document["journal"]["root_instance_id"], + **{ + key: copy.deepcopy(record[key]) + for key in ( + "effect_id", + "operation_token", + "attempt_fence", + "handler_reference", + "destination_binding_digest", + ) + }, + } + def _connect(self) -> sqlite3.Connection: connection = sqlite3.connect(self.path, isolation_level=None, timeout=30) connection.execute("PRAGMA journal_mode=WAL") @@ -1551,64 +1610,26 @@ def claim( trusted_now: str, deduplication_evidence: Mapping[str, Any] | None = None, ) -> dict[str, Any]: - final_proof: tuple[dict[str, Any], str, dict[str, Any]] | None = None + final_proof: tuple[dict[str, Any], dict[str, Any]] | None = None def verify_before_commit() -> None: - if final_proof is None: - return - reference, destination, proof = final_proof - handler = self._verified_handler(reference, destination) - VerifiedNativeHandler.verify_deduplication_evidence( - handler, reference, destination, proof - ) + if final_proof is not None: + context, proof = final_proof + SQLiteCommittedEffectHost._verify_retry_evidence(self, context, proof) + if self.trusted_clock is None: + raise EffectError("stale_attempt_fence") + self._check_live_clock(self.trusted_clock, _now(expires_at)) def change(document: dict[str, Any]) -> dict[str, Any]: nonlocal final_proof record = _record(document["journal"], effect_id) proof = None - if deduplication_evidence is not None: - if not isinstance(deduplication_evidence, Mapping): - raise EffectError("host_capability_mismatch") - proof = copy.deepcopy(dict(deduplication_evidence)) - fields = { - "scope_identity", - "effect_id", - "destination_binding_digest", - "first_attempt_receipt_bytes_base64", - "repeat_attempt_receipt_bytes_base64", - } - if ( - set(proof) != fields - or proof["scope_identity"] != document["journal"]["scope_identity"] - or proof["effect_id"] != effect_id - or proof["destination_binding_digest"] != record["destination_binding_digest"] - ): - raise EffectError("host_capability_mismatch") - try: - first = base64.b64decode( - proof["first_attempt_receipt_bytes_base64"], validate=True - ) - repeated = base64.b64decode( - proof["repeat_attempt_receipt_bytes_base64"], validate=True - ) - if not first or first != repeated: - raise ValueError("destination receipts differ") - except (ValueError, TypeError) as exc: - raise EffectError("host_capability_mismatch") from exc - handler = self._verified_handler( - record["handler_reference"], record["destination_binding_digest"] - ) - VerifiedNativeHandler.verify_deduplication_evidence( - handler, - record["handler_reference"], - record["destination_binding_digest"], - proof, - ) - final_proof = ( - copy.deepcopy(record["handler_reference"]), - record["destination_binding_digest"], - copy.deepcopy(proof), + if record["attempt_fence"] != "0" or deduplication_evidence is not None: + context = SQLiteCommittedEffectHost._retry_context(document, record) + proof = SQLiteCommittedEffectHost._verify_retry_evidence( + self, context, deduplication_evidence ) + final_proof = (context, proof) claim = _issue_effect_claim( document, effect_id, @@ -1621,6 +1642,7 @@ def change(document: dict[str, Any]) -> dict[str, Any]: if proof is not None: document.setdefault("destination_evidence", {}).setdefault(effect_id, []).append( { + "decision": "claim", "root_instance_id": root, "operation_token": record["operation_token"], "handler_reference": copy.deepcopy(record["handler_reference"]), @@ -1893,6 +1915,7 @@ def submit_result( scope: str, epoch: str, trusted_now: str, + deduplication_evidence: Mapping[str, Any] | None = None, ) -> dict[str, Any]: if ( not isinstance(request, Mapping) @@ -1912,7 +1935,10 @@ def submit_result( scope=scope, epoch=epoch, trusted_now=trusted_now, + deduplication_evidence=deduplication_evidence, ) + if response["status"] == "committed": + return response if request["outcome_kind"] in _OUTCOMES: # The authenticated outcome is already durable. Admission is a # separate host-owned transaction; its failure cannot erase it. @@ -1945,16 +1971,145 @@ def _record_result( scope: str, epoch: str, trusted_now: str, + deduplication_evidence: Mapping[str, Any] | None = None, ) -> dict[str, Any]: """Private durable-outcome stage; never a completed public result.""" + saved = self._saved_result_response(root, request, principal, scope, epoch) + if saved is not None: + return saved + final_proof: tuple[dict[str, Any], dict[str, Any]] | None = None + + def change(document: dict[str, Any]) -> dict[str, Any]: + nonlocal final_proof + key = request["effect_id"] + ":" + request["attempt_fence"] + fresh_retry = ( + request["outcome_kind"] == "retryable_failure" + and key not in document["result_requests"] + ) + proof = None + if fresh_retry: + record = self._authorize( + document, + request["effect_id"], + request["operation_token"], + request["attempt_fence"], + principal, + scope, + epoch, + trusted_now, + ) + context = SQLiteCommittedEffectHost._retry_context(document, record) + proof = SQLiteCommittedEffectHost._verify_retry_evidence( + self, context, deduplication_evidence + ) + final_proof = (context, proof) + response = self._submit( + document, + request, + principal, + scope, + epoch, + trusted_now, + retry_verified=proof is not None, + ) + if proof is not None: + document["result_requests"][key] = copy.deepcopy(dict(request)) + document.setdefault("result_responses", {})[key] = copy.deepcopy(response) + document.setdefault("result_disclosure", {})[key] = { + "principal": principal, + "scope": scope, + "epoch": epoch, + } + document.setdefault("destination_evidence", {}).setdefault( + request["effect_id"], [] + ).append( + { + "decision": "report", + "attempt_fence": request["attempt_fence"], + "evidence": proof, + } + ) + return response + + def verify_before_commit() -> None: + if final_proof is not None: + SQLiteCommittedEffectHost._verify_retry_evidence(self, *final_proof) + return self._transact( root, - lambda document: self._submit(document, request, principal, scope, epoch, trusted_now), + change, expected_epoch=epoch, authority_mutation=self._mirror_authority, worker_guard=(request["effect_id"], principal, request["attempt_fence"], trusted_now), + before_native_commit=verify_before_commit, ) + def _saved_result_response( + self, root: str, request: Mapping[str, Any], principal: str, scope: str, epoch: str + ) -> dict[str, Any] | None: + """Disclose retained native evidence without restoring today's executable. + + This read never grants a lease, invokes a verifier, or changes a receipt. + A retained request conflict must not fall through to a fresh decision. + """ + with self._connect() as connection: + connection.execute("BEGIN") + row = connection.execute( + "SELECT document FROM determa_committed_effects WHERE root_instance_id=?", (root,) + ).fetchone() + if row is None: + raise EffectError("wrong_root") + document = json.loads(row[0]) + key = request["effect_id"] + ":" + request["attempt_fence"] + prior = document["result_requests"].get(key) + response = document.get("result_responses", {}).get(key) + if prior is None or response is None: + return None + if self.authority_scope is not None: + authority = connection.execute( + "SELECT ledger FROM determa_scope_authority WHERE scope_identity=?", + (self.authority_scope,), + ).fetchone() + if authority is None: + raise EffectError("unauthorized_scope") + _validate_authority_pair(json.loads(authority[0]), document) + validate_journal(document["checkpoint"], document["journal"]) + disclosure = document.get("result_disclosure", {}).get(key) + if disclosure is None: + self._authorize( + document, + request["effect_id"], + request["operation_token"], + request["attempt_fence"], + principal, + scope, + epoch, + "0", + allow_replay=True, + ) + elif disclosure != {"principal": principal, "scope": scope, "epoch": epoch}: + raise EffectError("unauthorized_scope") + if prior != dict(request): + raise EffectError("effect_result_conflict") + record = _record(document["journal"], request["effect_id"]) + if _retained_report(record, request) is None: + raise EffectError("effect_result_conflict") + if request["outcome_kind"] == "retryable_failure" and not any( + entry.get("decision") == "report" + and entry.get("attempt_fence") == request["attempt_fence"] + and isinstance(entry.get("evidence"), dict) + and all( + entry["evidence"].get(key) == value + for key, value in { + **self._retry_context(document, record), + "attempt_fence": request["attempt_fence"], + }.items() + ) + for entry in document.get("destination_evidence", {}).get(request["effect_id"], []) + ): + raise EffectError("host_capability_mismatch") + return _validate_request(response, "effect-result-response-v1.schema.json") + def _submit( self, document: dict[str, Any], @@ -1963,6 +2118,8 @@ def _submit( scope: str, epoch: str, trusted_now: str, + *, + retry_verified: bool = False, ) -> dict[str, Any]: journal, checkpoint = document["journal"], document["checkpoint"] effect_id, fence, kind = ( @@ -2010,6 +2167,8 @@ def _submit( ) if record["invocation_state"] != "leased": raise EffectError("effect_not_outstanding") + if kind == "retryable_failure" and not retry_verified: + raise EffectError("host_capability_mismatch") payload = copy.deepcopy(request["payload"]) if kind in _OUTCOMES: self._result_delivery(checkpoint, record, kind, payload) @@ -2032,6 +2191,11 @@ def _submit( } record["attempt_records"].append(report) document["result_requests"][key] = copy.deepcopy(dict(request)) + document.setdefault("result_disclosure", {})[key] = { + "principal": principal, + "scope": scope, + "epoch": epoch, + } if kind not in _OUTCOMES: record["invocation_state"] = "unclaimed" if kind == "retryable_failure" else "ambiguous" _bump(journal) diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index de216ef..3cd0db0 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -101,6 +101,67 @@ def verify(source, actual): ) +def installed_retry_handler(host, root): + """Authenticate scoped native receipt bytes independently of worker input.""" + import base64 + import sqlite3 + + saved = host.snapshot(root) + record = saved["journal"]["effect_records"][0] + context = SQLiteCommittedEffectHost._retry_context(saved, record) + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + destination = host.path + ".destination" + receipt = json.dumps( + {key: value for key, value in context.items() if key != "attempt_fence"}, sort_keys=True + ).encode() + with sqlite3.connect(destination) as connection: + connection.execute("CREATE TABLE receipts (scope TEXT, effect TEXT, receipt BLOB)") + connection.execute( + "INSERT INTO receipts VALUES (?, ?, ?)", + (context["scope_identity"], record["effect_id"], receipt), + ) + observations = [] + + def verify_receipts(evidence): + observations.append(copy.deepcopy(evidence)) + with sqlite3.connect(destination) as connection: + row = connection.execute( + "SELECT receipt FROM receipts WHERE scope=? AND effect=?", + (evidence["scope_identity"], evidence["effect_id"]), + ).fetchone() + return ( + row is not None + and all( + base64.b64decode(evidence[name], validate=True) == row[0] + for name in ( + "first_attempt_receipt_bytes_base64", + "repeat_attempt_receipt_bytes_base64", + ) + ) + and json.loads(row[0]) + == { + key: value + for key, value in evidence.items() + if key + not in { + "attempt_fence", + "first_attempt_receipt_bytes_base64", + "repeat_attempt_receipt_bytes_base64", + } + } + ) + + host = installed_test_handler(host, lambda *_: {}, verify_receipts) + proof = { + **context, + "first_attempt_receipt_bytes_base64": base64.b64encode(receipt).decode(), + "repeat_attempt_receipt_bytes_base64": base64.b64encode(receipt).decode(), + } + return host, proof, observations + + def host_fixture(tmp_path, observer=None): bundle = load_bundle((CASE / "machine.yaml").read_text()) resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) @@ -412,8 +473,8 @@ def test_worker_result_crossing_expiry_rolls_back_outcome_before_commit(tmp_path submit = host._submit observed = [] - def expires_during_submission(*args): - response = submit(*args) + def expires_during_submission(*args, **kwargs): + response = submit(*args, **kwargs) observed.append(args[0]["journal"]["effect_records"][0]["invocation_state"]) current_time[0] = read("data/active-claim.json")["expires_at"] return response @@ -820,8 +881,15 @@ def test_seed_preserves_historical_claim_without_reviving_it(tmp_path): "payload": ["map", []], } host.trusted_clock = lambda: "0" + host, proof, _ = installed_retry_handler(host, root) response = host.submit_result( - root, report, principal=claim["worker_principal"], scope=scope, epoch="0", trusted_now="0" + root, + report, + principal=claim["worker_principal"], + scope=scope, + epoch="0", + trusted_now="0", + deduplication_evidence=proof, ) assert response["status"] == "report_recorded" saved = host.snapshot(root) @@ -835,7 +903,13 @@ def test_seed_preserves_historical_claim_without_reviving_it(tmp_path): assert authority.inspect(scope)["active_claims"] == [] assert host.snapshot(root)["claims"][record["effect_id"]] == claim new_claim = host.claim( - root, record["effect_id"], "worker-b", "0", expires_at="30", trusted_now="20" + root, + record["effect_id"], + "worker-b", + "0", + expires_at="30", + trusted_now="20", + deduplication_evidence=proof, ) assert new_claim["attempt_fence"] == "2" @@ -929,7 +1003,9 @@ def test_multi_effect_recovery_commits_one_journal_revision_per_effect(tmp_path) journal["checkpoint_revision"] = checkpoint["revision"] journal["checkpoint_digest"] = checkpoint["execution_checkpoint_digest"] journal = seal_journal(journal) - host = SQLiteCommittedEffectHost(tmp_path / "two.sqlite", original.resolver, {}, None) + host = SQLiteCommittedEffectHost( + tmp_path / "two.sqlite", original.resolver, {}, None, trusted_clock=lambda: "0" + ) host.setup_schema() host.seed(checkpoint, journal) for record in journal["effect_records"]: @@ -1306,6 +1382,10 @@ def test_historical_result_evidence_survives_processing(tmp_path): "fabricated", "wrong_work", "wrong_destination", + "wrong_root", + "wrong_token", + "wrong_fence", + "wrong_handler", "no_native_receipt", "unavailable", ], @@ -1316,7 +1396,9 @@ def test_ambiguous_retry_requires_verified_native_destination_evidence(tmp_path, original, root, request, _ = host_fixture(tmp_path) journal = read("data/ambiguous-journal.json") - host = SQLiteCommittedEffectHost(tmp_path / "ambiguous.sqlite", original.resolver, {}, None) + host = SQLiteCommittedEffectHost( + tmp_path / "ambiguous.sqlite", original.resolver, {}, None, trusted_clock=lambda: "0" + ) host.setup_schema() host.seed(read("pending-checkpoint.json"), journal) record = journal["effect_records"][0] @@ -1365,6 +1447,11 @@ def verify_receipts(evidence): host = installed_test_handler(host, lambda *_: {}, verify_receipts) encoded = base64.b64encode(receipt_bytes).decode() proof = { + "kind": "destination_deduplication", + "root_instance_id": root, + "operation_token": record["operation_token"], + "attempt_fence": record["attempt_fence"], + "handler_reference": copy.deepcopy(record["handler_reference"]), "scope_identity": journal["scope_identity"], "effect_id": record["effect_id"], "destination_binding_digest": record["destination_binding_digest"], @@ -1381,6 +1468,14 @@ def verify_receipts(evidence): proof["effect_id"] = "sha256:" + "0" * 64 elif damage == "wrong_destination": proof["destination_binding_digest"] = "sha256:" + "0" * 64 + elif damage == "wrong_root": + proof["root_instance_id"] = "other-root" + elif damage == "wrong_token": + proof["operation_token"] = "other-token" + elif damage == "wrong_fence": + proof["attempt_fence"] = "0" + elif damage == "wrong_handler": + proof["handler_reference"]["identifier"] = "other-handler" before = host.snapshot(root) if damage is not None: with pytest.raises(EffectError, match="host_capability_mismatch"): @@ -1430,16 +1525,166 @@ def test_ambiguous_retry_does_not_accept_legacy_boolean_keyword(tmp_path): ) +@pytest.mark.parametrize( + "damage", + [ + None, + "missing", + "boolean", + "fabricated", + "wrong_root", + "wrong_token", + "wrong_fence", + "wrong_handler", + "unavailable", + ], +) +def test_retryable_report_requires_native_original_invocation_proof(tmp_path, damage): + import base64 + + host, root, request, context = host_fixture(tmp_path) + host, proof, checks = installed_retry_handler(host, root) + report = {**request, "outcome_kind": "retryable_failure", "payload": ["map", []]} + if damage == "missing": + proof = None + elif damage == "boolean": + proof = True + elif damage == "fabricated": + proof["first_attempt_receipt_bytes_base64"] = proof[ + "repeat_attempt_receipt_bytes_base64" + ] = base64.b64encode(b"fabricated equal receipts").decode() + elif damage == "wrong_root": + proof["root_instance_id"] = "other-root" + elif damage == "wrong_token": + proof["operation_token"] = "other-token" + elif damage == "wrong_fence": + proof["attempt_fence"] = "0" + elif damage == "wrong_handler": + proof["handler_reference"]["identifier"] = "other-handler" + elif damage == "unavailable": + host.handler._configured._provider.proof_verifier = None + before = host.snapshot(root) + response = host.submit_result(root, report, **context, deduplication_evidence=proof) + if damage is not None: + assert response["error_code"] == "host_capability_mismatch" + assert host.snapshot(root) == before + else: + assert response["status"] == "report_recorded" + assert response["attempt_report"]["reason"] == "destination_deduplication_proven" + assert checks == [proof, proof] + retained = host.snapshot(root)["destination_evidence"][request["effect_id"]][0] + assert retained == { + "decision": "report", + "attempt_fence": request["attempt_fence"], + "evidence": proof, + } + + +def test_safe_report_replay_and_later_claim_never_reuse_old_verification(tmp_path): + host, root, request, context = host_fixture(tmp_path) + host, proof, checks = installed_retry_handler(host, root) + report = {**request, "outcome_kind": "retryable_failure", "payload": ["map", []]} + first = host.submit_result(root, report, **context, deduplication_evidence=proof) + assert first["status"] == "report_recorded" + before = host.snapshot(root) + with pytest.raises(EffectError, match="host_capability_mismatch"): + host.claim( + root, + request["effect_id"], + "worker-b", + context["epoch"], + expires_at="10", + trusted_now="0", + ) + assert host.snapshot(root) == before + host.claim( + root, + request["effect_id"], + "worker-b", + context["epoch"], + expires_at="10", + trusted_now="0", + deduplication_evidence=proof, + ) + assert len(checks) == 4 + host.handler._configured._provider.proof_verifier = None + host.resolver = MemoryArtifactResolver(definitions={}) + with host._connect() as connection: + before_bytes = connection.execute( + "SELECT document FROM determa_committed_effects" + ).fetchone()[0] + assert host.submit_result(root, report, **context) == first + assert len(checks) == 4 + changed = {**report, "payload": ["map", [["changed", ["boolean", True]]]]} + assert host.submit_result(root, changed, **context)["error_code"] == "effect_result_conflict" + with host._connect() as connection: + assert ( + connection.execute("SELECT document FROM determa_committed_effects").fetchone()[0] + == before_bytes + ) + + +def test_authority_noninitial_native_claim_cannot_bypass_retry_verifier(tmp_path): + authority, host, scope, root, record = authority_effect_fixture(tmp_path) + command, invocation = authority_claim_request(authority, scope, root, record) + claim = json.loads(authority.perform(command, invocation))["claim"] + host.trusted_clock = lambda: "0" + host, proof, checks = installed_retry_handler(host, root) + report = { + "effect_id": record["effect_id"], + "operation_token": record["operation_token"], + "attempt_fence": claim["attempt_fence"], + "outcome_kind": "retryable_failure", + "payload": ["map", []], + } + assert ( + host.submit_result( + root, + report, + principal=claim["worker_principal"], + scope=scope, + epoch="0", + trusted_now="0", + deduplication_evidence=proof, + )["status"] + == "report_recorded" + ) + before = host.snapshot(root), authority.inspect(scope) + command, invocation = authority_claim_request( + authority, + scope, + root, + record, + expected="1", + operation="second-claim-without-native-verification", + ) + response = json.loads(authority.perform(command, invocation)) + assert response["status"] == "rejected" + assert response["error_code"] == "host_capability_mismatch" + assert (host.snapshot(root), authority.inspect(scope)) == before + assert len(checks) == 2 + + @pytest.mark.parametrize("damage", [None, "unavailable", "method", "configuration", "instance"]) -def test_retry_verifier_is_rechecked_after_native_sql_staging(tmp_path, monkeypatch, damage): +@pytest.mark.parametrize("operation", ["claim", "report"]) +def test_retry_verifier_is_rechecked_after_native_sql_staging( + tmp_path, monkeypatch, damage, operation +): import base64 import sqlite3 - original, root, request, _ = host_fixture(tmp_path) - journal = read("data/ambiguous-journal.json") + original, root, request, context = host_fixture(tmp_path) + journal = read( + "data/ambiguous-journal.json" if operation == "claim" else "data/leased-journal.json" + ) host = SQLiteCommittedEffectHost(tmp_path / "retry-staging.sqlite", original.resolver, {}, None) host.setup_schema() - host.seed(read("pending-checkpoint.json"), journal) + host.seed( + read("pending-checkpoint.json"), + journal, + read("data/active-claim.json") if operation == "report" else None, + ) + host.trusted_clock = lambda: "0" record = journal["effect_records"][0] host.route.update( {key: record[key] for key in ("handler_reference", "destination_binding_digest")} @@ -1468,6 +1713,11 @@ def verify_receipts(evidence): host = installed_test_handler(host, lambda *_: {}, verify_receipts) encoded = base64.b64encode(receipt).decode() proof = { + "kind": "destination_deduplication", + "root_instance_id": root, + "operation_token": record["operation_token"], + "attempt_fence": record["attempt_fence"], + "handler_reference": copy.deepcopy(record["handler_reference"]), "scope_identity": journal["scope_identity"], "effect_id": record["effect_id"], "destination_binding_digest": record["destination_binding_digest"], @@ -1489,7 +1739,11 @@ def execute(self, sql, parameters=()): ) .fetchone()[0] ) - assert staged["claims"][request["effect_id"]]["attempt_fence"] == "2" + assert staged["claims"][request["effect_id"]]["attempt_fence"] == ( + "2" if operation == "claim" else "1" + ) + if operation == "report": + assert staged["journal"]["effect_records"][0]["invocation_state"] == "unclaimed" state["staged"] = True if damage == "method": monkeypatch.setattr( @@ -1507,17 +1761,80 @@ def execute(self, sql, parameters=()): lambda: sqlite3.connect(host.path, isolation_level=None, factory=StagingConnection), ) arguments = {"expires_at": "10", "trusted_now": "0", "deduplication_evidence": proof} + + def execute(): + if operation == "claim": + return host.claim(root, request["effect_id"], "worker", "0", **arguments) + response = host.submit_result( + root, + {**request, "outcome_kind": "retryable_failure", "payload": ["map", []]}, + **context, + deduplication_evidence=proof, + ) + if response["status"] == "rejected": + raise EffectError(response["error_code"]) + return response + if damage is None: - claim = host.claim(root, request["effect_id"], "worker", "0", **arguments) - assert claim["attempt_fence"] == "2" + response = execute() + assert response["attempt_fence"] == ("2" if operation == "claim" else "1") assert state["staged"] and state["checks"] >= 2 else: with pytest.raises(EffectError, match="host_capability_mismatch"): - host.claim(root, request["effect_id"], "worker", "0", **arguments) + execute() assert state["staged"] assert host.snapshot(root) == before +@pytest.mark.parametrize("clock", ["expired", "unavailable", "invalid", "missing"]) +def test_initial_claim_requires_fresh_clock_after_actual_sql_staging(tmp_path, monkeypatch, clock): + import sqlite3 + + original, root, request, _ = host_fixture(tmp_path) + state = {"staged": False} + + def now(): + if not state["staged"]: + return "0" + if clock == "unavailable": + raise RuntimeError("clock unavailable") + return "10" if clock == "expired" else "invalid" + + host = SQLiteCommittedEffectHost( + tmp_path / "claim-clock.sqlite", original.resolver, {}, None, trusted_clock=now + ) + host.setup_schema() + host.seed(read("pending-checkpoint.json"), read("data/unclaimed-journal.json")) + before = host.snapshot(root) + + class StagingConnection(sqlite3.Connection): + def execute(self, sql, parameters=()): + result = super().execute(sql, parameters) + if sql.startswith("UPDATE determa_committed_effects SET document"): + row = ( + super() + .execute( + "SELECT document FROM determa_committed_effects WHERE root_instance_id=?", + (root,), + ) + .fetchone() + ) + assert json.loads(row[0])["claims"][request["effect_id"]]["attempt_fence"] == "1" + state["staged"] = True + if clock == "missing": + host.trusted_clock = None + return result + + monkeypatch.setattr( + host, + "_connect", + lambda: sqlite3.connect(host.path, isolation_level=None, factory=StagingConnection), + ) + with pytest.raises(EffectError, match="stale_attempt_fence"): + host.claim(root, request["effect_id"], "worker", "0", expires_at="10", trusted_now="0") + assert state["staged"] and host.snapshot(root) == before + + def test_historical_admission_survives_actual_root_completion_and_tombstone(tmp_path): import yaml From 40140ad3d7ba7e156e8fd33bfad37535d6d43bb5 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Tue, 6 Oct 2026 07:06:22 +0000 Subject: [PATCH 26/32] fix(effects): refuse non-string native retry receipts before verification --- src/determa/state/effects.py | 2 ++ tests/test_committed_effects.py | 37 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index cdfa8bb..74a53a7 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -1064,6 +1064,8 @@ def _verify_retry_evidence( proof = copy.deepcopy(dict(evidence)) if any(proof[key] != value for key, value in context.items()): raise EffectError("host_capability_mismatch") + if any(type(proof[field]) is not str for field in receipt_fields): + raise EffectError("host_capability_mismatch") try: first = base64.b64decode(proof["first_attempt_receipt_bytes_base64"], validate=True) repeated = base64.b64decode(proof["repeat_attempt_receipt_bytes_base64"], validate=True) diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 3cd0db0..f48d032 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -1580,6 +1580,43 @@ def test_retryable_report_requires_native_original_invocation_proof(tmp_path, da } +@pytest.mark.parametrize("operation", ["report", "claim"]) +@pytest.mark.parametrize("receipt_type", [bytes, bytearray]) +def test_nonstring_native_receipts_refuse_before_checker_or_mutation( + tmp_path, operation, receipt_type +): + host, root, request, context = host_fixture(tmp_path) + if operation == "claim": + host = SQLiteCommittedEffectHost( + tmp_path / "nonstring-claim.sqlite", host.resolver, {}, None, trusted_clock=lambda: "0" + ) + host.setup_schema() + host.seed(read("pending-checkpoint.json"), read("data/ambiguous-journal.json")) + host, proof, checks = installed_retry_handler(host, root) + for field in ("first_attempt_receipt_bytes_base64", "repeat_attempt_receipt_bytes_base64"): + proof[field] = receipt_type(proof[field].encode("ascii")) + before = host.snapshot(root) + if operation == "report": + report = {**request, "outcome_kind": "retryable_failure", "payload": ["map", []]} + assert ( + host.submit_result(root, report, **context, deduplication_evidence=proof)["error_code"] + == "host_capability_mismatch" + ) + else: + with pytest.raises(EffectError, match="host_capability_mismatch"): + host.claim( + root, + request["effect_id"], + "worker", + context["epoch"], + expires_at="10", + trusted_now="0", + deduplication_evidence=proof, + ) + assert checks == [] + assert host.snapshot(root) == before + + def test_safe_report_replay_and_later_claim_never_reuse_old_verification(tmp_path): host, root, request, context = host_fixture(tmp_path) host, proof, checks = installed_retry_handler(host, root) From e943f1d7cdcc63bf1ddae3127a513d543175e6a3 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Tue, 6 Oct 2026 10:37:53 +0000 Subject: [PATCH 27/32] WIP: seal the final native retry clock read --- conformance/effects_adapter.py | 3 + src/determa/state/effects.py | 57 ++++++++++++++--- src/determa/state/native_clock.py | 53 +++++++++++++++ tests/test_committed_effects.py | 103 +++++++++++++++++++++++++++++- 4 files changed, 205 insertions(+), 11 deletions(-) create mode 100644 src/determa/state/native_clock.py diff --git a/conformance/effects_adapter.py b/conformance/effects_adapter.py index 12527f1..a781f0b 100644 --- a/conformance/effects_adapter.py +++ b/conformance/effects_adapter.py @@ -546,6 +546,8 @@ def verify_native(source: Any, selected: Any) -> bool: }, ) installed_handler = VerifiedNativeHandler(handler_registry, configured_handler) + from determa.state.native_clock import NativeCommitClock + host = SQLiteCommittedEffectHost( _PATH, resolver, @@ -554,6 +556,7 @@ def verify_native(source: Any, selected: Any) -> bool: authority_scope=_AUTHORITY_SCOPE, core_observer=observe_core, trusted_clock=lambda: payload["auth_context"]["trusted_host_now"], + commit_clock=NativeCommitClock.controlled(int(payload["auth_context"]["trusted_host_now"])), ) # Install the vector's pre-existing authority facts as trusted test setup. # The production seed path may restore these claims, but cannot issue them. diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 74a53a7..60652ea 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -27,6 +27,7 @@ from .errors import ArtifactError from .extensions import ConfiguredExtension, ExtensionError, ExtensionRegistry from .host import outbox_intent_digest +from .native_clock import NativeCommitClock, _read_native_clock from .wire import ( ArtifactResolver, _schema_registry, @@ -1028,6 +1029,7 @@ def __init__( authority_scope: str | None = None, core_observer: Callable[[str, str, Mapping[str, Any]], None] | None = None, trusted_clock: Callable[[], str] | None = None, + commit_clock: NativeCommitClock | None = None, ) -> None: if handler is not None and type(handler) is not VerifiedNativeHandler: raise EffectError("host_capability_mismatch") @@ -1039,6 +1041,32 @@ def __init__( self.authority_scope = authority_scope self.core_observer = core_observer self.trusted_clock = trusted_clock + if commit_clock is not None and type(commit_clock) is not NativeCommitClock: + raise EffectError("host_capability_mismatch") + self.commit_clock = commit_clock + self._installed_commit_clock = commit_clock + + def _retry_commit_clock(self) -> tuple[NativeCommitClock, int]: + clock = self.commit_clock + if type(clock) is not NativeCommitClock or clock is not self._installed_commit_clock: + raise EffectError("host_capability_mismatch") + try: + return clock, _read_native_clock(clock) + except ValueError as exc: + raise EffectError("stale_attempt_fence") from exc + + def _check_retry_commit_clock( + self, guard: tuple[NativeCommitClock, int], deadline: int + ) -> None: + clock, start = guard + if clock is not self.commit_clock or clock is not self._installed_commit_clock: + raise EffectError("host_capability_mismatch") + try: + now = _read_native_clock(clock) + except ValueError as exc: + raise EffectError("stale_attempt_fence") from exc + if now < start or now >= deadline: + raise EffectError("stale_attempt_fence") def _verified_handler( self, reference: Mapping[str, Any], destination: str @@ -1447,12 +1475,11 @@ def _transact( "UPDATE determa_committed_effects SET document = ? WHERE root_instance_id = ?", (canonical_bytes(document), root), ) - if before_native_commit is not None: - before_native_commit() if expiry_guard is not None: - # Keep the original deadline even when this transaction closes - # or revokes the claim. Worker rights must still hold at commit. + # Arbitrary legacy clock callbacks run before native proof. self._check_live_clock(*expiry_guard) + if before_native_commit is not None: + before_native_commit() connection.commit() return value @@ -1613,21 +1640,27 @@ def claim( deduplication_evidence: Mapping[str, Any] | None = None, ) -> dict[str, Any]: final_proof: tuple[dict[str, Any], dict[str, Any]] | None = None + clock_guard: tuple[NativeCommitClock, int] | None = None def verify_before_commit() -> None: - if final_proof is not None: - context, proof = final_proof - SQLiteCommittedEffectHost._verify_retry_evidence(self, context, proof) if self.trusted_clock is None: raise EffectError("stale_attempt_fence") self._check_live_clock(self.trusted_clock, _now(expires_at)) + if final_proof is not None: + context, proof = final_proof + SQLiteCommittedEffectHost._verify_retry_evidence(self, context, proof) + assert clock_guard is not None + SQLiteCommittedEffectHost._check_retry_commit_clock( + self, clock_guard, _now(expires_at) + ) def change(document: dict[str, Any]) -> dict[str, Any]: - nonlocal final_proof + nonlocal final_proof, clock_guard record = _record(document["journal"], effect_id) proof = None if record["attempt_fence"] != "0" or deduplication_evidence is not None: context = SQLiteCommittedEffectHost._retry_context(document, record) + clock_guard = SQLiteCommittedEffectHost._retry_commit_clock(self) proof = SQLiteCommittedEffectHost._verify_retry_evidence( self, context, deduplication_evidence ) @@ -1980,9 +2013,11 @@ def _record_result( if saved is not None: return saved final_proof: tuple[dict[str, Any], dict[str, Any]] | None = None + clock_guard: tuple[NativeCommitClock, int] | None = None + deadline: int | None = None def change(document: dict[str, Any]) -> dict[str, Any]: - nonlocal final_proof + nonlocal final_proof, clock_guard, deadline key = request["effect_id"] + ":" + request["attempt_fence"] fresh_retry = ( request["outcome_kind"] == "retryable_failure" @@ -2001,6 +2036,8 @@ def change(document: dict[str, Any]) -> dict[str, Any]: trusted_now, ) context = SQLiteCommittedEffectHost._retry_context(document, record) + clock_guard = SQLiteCommittedEffectHost._retry_commit_clock(self) + deadline = _now(document["claims"][request["effect_id"]]["expires_at"]) proof = SQLiteCommittedEffectHost._verify_retry_evidence( self, context, deduplication_evidence ) @@ -2036,6 +2073,8 @@ def change(document: dict[str, Any]) -> dict[str, Any]: def verify_before_commit() -> None: if final_proof is not None: SQLiteCommittedEffectHost._verify_retry_evidence(self, *final_proof) + assert clock_guard is not None and deadline is not None + SQLiteCommittedEffectHost._check_retry_commit_clock(self, clock_guard, deadline) return self._transact( root, diff --git a/src/determa/state/native_clock.py b/src/determa/state/native_clock.py new file mode 100644 index 0000000..e119f95 --- /dev/null +++ b/src/determa/state/native_clock.py @@ -0,0 +1,53 @@ +"""Host-owned nanosecond reads with no user callback at the commit boundary. + +Controlled clocks are explicit native test/host sources. Their installer owns +sample updates and availability; they do not advance or schedule themselves. +""" + +from __future__ import annotations + +from time import time_ns as _system_time_ns + + +class NativeCommitClock: + """Exact native clock handle, separate from portable request timestamps.""" + + __slots__ = ("_sample", "_system") + + def __init__(self) -> None: + self._system = True + self._sample: tuple[int, bool] = (0, False) + + @classmethod + def controlled(cls, now_ns: int) -> NativeCommitClock: + clock = cls() + clock._system = False + clock.set_native_sample(now_ns) + return clock + + def set_native_sample(self, now_ns: int, *, unavailable: bool = False) -> None: + if type(now_ns) is not int or not -(2**63) <= now_ns < 2**63: + raise ValueError("invalid native clock sample") + if type(unavailable) is not bool or self._system: + raise ValueError("controlled native clock required") + self._sample = (now_ns, unavailable) + + +def _read_native_clock(clock: NativeCommitClock) -> int: + # Exact types and class-owned reads prevent overridden methods, numeric + # conversions or an arbitrary supplied callable from running after proof. + if type(clock) is not NativeCommitClock: + raise ValueError("native clock identity differs") + system = object.__getattribute__(clock, "_system") + if system is True: + now = _system_time_ns() + elif system is False: + sample = object.__getattribute__(clock, "_sample") + if type(sample) is not tuple or len(sample) != 2 or sample[1] is not False: + raise ValueError("native clock unavailable") + now = sample[0] + else: + raise ValueError("native clock configuration differs") + if type(now) is not int or not -(2**63) <= now < 2**63: + raise ValueError("invalid native clock sample") + return now diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index f48d032..e75d801 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -13,6 +13,7 @@ seal_journal, validate_journal, ) +from determa.state.native_clock import NativeCommitClock CASE = conformance_root() / "conformance/profiles/committed-native-effects/effect-01-result" @@ -98,6 +99,13 @@ def verify(source, actual): handler, authority_scope=host.authority_scope, trusted_clock=host.trusted_clock, + commit_clock=( + host.commit_clock + if host.commit_clock is not None + else NativeCommitClock.controlled(int(host.trusted_clock())) + if host.trusted_clock is not None + else None + ), ) @@ -1702,7 +1710,24 @@ def test_authority_noninitial_native_claim_cannot_bypass_retry_verifier(tmp_path assert len(checks) == 2 -@pytest.mark.parametrize("damage", [None, "unavailable", "method", "configuration", "instance"]) +@pytest.mark.parametrize( + "damage", + [ + None, + "unavailable", + "method", + "configuration", + "instance", + "clock_verifier", + "clock_method", + "clock_configuration", + "clock_instance", + "proof_expiry", + "proof_clock_unavailable", + "proof_clock_regression", + "proof_clock_replaced", + ], +) @pytest.mark.parametrize("operation", ["claim", "report"]) def test_retry_verifier_is_rechecked_after_native_sql_staging( tmp_path, monkeypatch, damage, operation @@ -1732,9 +1757,21 @@ def test_retry_verifier_is_rechecked_after_native_sql_staging( connection.execute("CREATE TABLE receipts (receipt BLOB NOT NULL)") connection.execute("INSERT INTO receipts VALUES (?)", (receipt,)) state = {"staged": False, "checks": 0} + lease_deadline = ( + 10 if operation == "claim" else int(read("data/active-claim.json")["expires_at"]) + ) def verify_receipts(evidence): state["checks"] += 1 + if state["staged"]: + if damage == "proof_expiry": + host.commit_clock.set_native_sample(lease_deadline) + elif damage == "proof_clock_unavailable": + host.commit_clock.set_native_sample(0, unavailable=True) + elif damage == "proof_clock_regression": + host.commit_clock.set_native_sample(-1) + elif damage == "proof_clock_replaced": + host.commit_clock = NativeCommitClock.controlled(0) if state["staged"] and damage == "unavailable": raise RuntimeError("native verifier unavailable after staging") with sqlite3.connect(native_path) as connection: @@ -1747,6 +1784,21 @@ def verify_receipts(evidence): ) ) + def trusted_now(): + if state["staged"]: + if damage == "clock_verifier": + host.handler._configured._provider.proof_verifier = None + elif damage == "clock_method": + monkeypatch.setattr( + NativeTestProvider, "verify_deduplication_evidence", lambda *_: True + ) + elif damage == "clock_configuration": + host.handler._configured._configuration["instance_id"] = "clock-substituted" + elif damage == "clock_instance": + object.__setattr__(host.handler._configured, "_instance", {}) + return "0" + + host.trusted_clock = trusted_now host = installed_test_handler(host, lambda *_: {}, verify_receipts) encoded = base64.b64encode(receipt).decode() proof = { @@ -1817,7 +1869,12 @@ def execute(): assert response["attempt_fence"] == ("2" if operation == "claim" else "1") assert state["staged"] and state["checks"] >= 2 else: - with pytest.raises(EffectError, match="host_capability_mismatch"): + code = ( + "stale_attempt_fence" + if damage in {"proof_expiry", "proof_clock_unavailable", "proof_clock_regression"} + else "host_capability_mismatch" + ) + with pytest.raises(EffectError, match=code): execute() assert state["staged"] assert host.snapshot(root) == before @@ -2523,3 +2580,45 @@ def test_private_dispatch_start_loss_cannot_be_blessed_by_dispatch_or_authority( assert json.loads(authority.perform(command, invocation))["claim"] == claim assert json.loads(authority.perform(fresh_command, fresh_invocation))["status"] == "rejected" assert len(calls) == 1 + + +@pytest.mark.parametrize("operation", ["claim", "report"]) +def test_fresh_retry_refuses_a_callback_clock_without_a_sealed_native_reader(tmp_path, operation): + original, root, request, context = host_fixture(tmp_path) + if operation == "claim": + host = SQLiteCommittedEffectHost( + tmp_path / "callback-clock.sqlite", + original.resolver, + {}, + None, + trusted_clock=lambda: "0", + ) + host.setup_schema() + host.seed(read("pending-checkpoint.json"), read("data/ambiguous-journal.json")) + else: + host = original + host, proof, checks = installed_retry_handler(host, root) + host.commit_clock = None + before = host.snapshot(root) + if operation == "claim": + with pytest.raises(EffectError, match="host_capability_mismatch"): + host.claim( + root, + request["effect_id"], + "worker", + "0", + expires_at="10", + trusted_now="0", + deduplication_evidence=proof, + ) + else: + result = host.submit_result( + root, + {**request, "outcome_kind": "retryable_failure", "payload": ["map", []]}, + **context, + deduplication_evidence=proof, + ) + assert result["status"] == "rejected" + assert result["error_code"] == "host_capability_mismatch" + assert checks == [] + assert host.snapshot(root) == before From ce10166e33081fb54011c0047a6e6783a832d95e Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Tue, 6 Oct 2026 10:41:52 +0000 Subject: [PATCH 28/32] WIP: pin native clock slots and source mode at retry commit --- src/determa/state/effects.py | 25 +++++++++++++++------- src/determa/state/native_clock.py | 35 ++++++++++++++++++++++--------- tests/test_committed_effects.py | 27 ++++++++++++++++++++++-- 3 files changed, 67 insertions(+), 20 deletions(-) diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 60652ea..b2b4bd7 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -27,7 +27,7 @@ from .errors import ArtifactError from .extensions import ConfiguredExtension, ExtensionError, ExtensionRegistry from .host import outbox_intent_digest -from .native_clock import NativeCommitClock, _read_native_clock +from .native_clock import NativeCommitClock, _native_clock_mode, _read_native_clock from .wire import ( ArtifactResolver, _schema_registry, @@ -1045,24 +1045,33 @@ def __init__( raise EffectError("host_capability_mismatch") self.commit_clock = commit_clock self._installed_commit_clock = commit_clock + try: + self._installed_commit_clock_mode = ( + _native_clock_mode(commit_clock) if commit_clock is not None else None + ) + except ValueError as exc: + raise EffectError("host_capability_mismatch") from exc - def _retry_commit_clock(self) -> tuple[NativeCommitClock, int]: + def _retry_commit_clock(self) -> tuple[NativeCommitClock, int, bool]: clock = self.commit_clock if type(clock) is not NativeCommitClock or clock is not self._installed_commit_clock: raise EffectError("host_capability_mismatch") try: - return clock, _read_native_clock(clock) + mode = self._installed_commit_clock_mode + if type(mode) is not bool: + raise ValueError("native clock mode absent") + return clock, _read_native_clock(clock, mode=mode), mode except ValueError as exc: raise EffectError("stale_attempt_fence") from exc def _check_retry_commit_clock( - self, guard: tuple[NativeCommitClock, int], deadline: int + self, guard: tuple[NativeCommitClock, int, bool], deadline: int ) -> None: - clock, start = guard + clock, start, mode = guard if clock is not self.commit_clock or clock is not self._installed_commit_clock: raise EffectError("host_capability_mismatch") try: - now = _read_native_clock(clock) + now = _read_native_clock(clock, mode=mode) except ValueError as exc: raise EffectError("stale_attempt_fence") from exc if now < start or now >= deadline: @@ -1640,7 +1649,7 @@ def claim( deduplication_evidence: Mapping[str, Any] | None = None, ) -> dict[str, Any]: final_proof: tuple[dict[str, Any], dict[str, Any]] | None = None - clock_guard: tuple[NativeCommitClock, int] | None = None + clock_guard: tuple[NativeCommitClock, int, bool] | None = None def verify_before_commit() -> None: if self.trusted_clock is None: @@ -2013,7 +2022,7 @@ def _record_result( if saved is not None: return saved final_proof: tuple[dict[str, Any], dict[str, Any]] | None = None - clock_guard: tuple[NativeCommitClock, int] | None = None + clock_guard: tuple[NativeCommitClock, int, bool] | None = None deadline: int | None = None def change(document: dict[str, Any]) -> dict[str, Any]: diff --git a/src/determa/state/native_clock.py b/src/determa/state/native_clock.py index e119f95..6903d83 100644 --- a/src/determa/state/native_clock.py +++ b/src/determa/state/native_clock.py @@ -33,21 +33,36 @@ def set_native_sample(self, now_ns: int, *, unavailable: bool = False) -> None: self._sample = (now_ns, unavailable) -def _read_native_clock(clock: NativeCommitClock) -> int: - # Exact types and class-owned reads prevent overridden methods, numeric - # conversions or an arbitrary supplied callable from running after proof. - if type(clock) is not NativeCommitClock: +# Retain actual native member descriptors, not a dynamic attribute access which +# could invoke a property installed by a callback after proof verification. +_NATIVE_CLOCK_TYPE = NativeCommitClock +_SYSTEM_SLOT = NativeCommitClock.__dict__["_system"] +_SAMPLE_SLOT = NativeCommitClock.__dict__["_sample"] + + +def _native_clock_mode(clock: NativeCommitClock) -> bool: + if type(clock) is not _NATIVE_CLOCK_TYPE: raise ValueError("native clock identity differs") - system = object.__getattribute__(clock, "_system") - if system is True: + namespace = type.__getattribute__(_NATIVE_CLOCK_TYPE, "__dict__") + if namespace.get("_system") is not _SYSTEM_SLOT or namespace.get("_sample") is not _SAMPLE_SLOT: + raise ValueError("native clock descriptors differ") + mode = _SYSTEM_SLOT.__get__(clock, _NATIVE_CLOCK_TYPE) + if type(mode) is not bool: + raise ValueError("native clock configuration differs") + return mode + + +def _read_native_clock(clock: NativeCommitClock, *, mode: bool) -> int: + system = _native_clock_mode(clock) + if system is not mode: + raise ValueError("native clock source differs") + if system: now = _system_time_ns() - elif system is False: - sample = object.__getattribute__(clock, "_sample") + else: + sample = _SAMPLE_SLOT.__get__(clock, _NATIVE_CLOCK_TYPE) if type(sample) is not tuple or len(sample) != 2 or sample[1] is not False: raise ValueError("native clock unavailable") now = sample[0] - else: - raise ValueError("native clock configuration differs") if type(now) is not int or not -(2**63) <= now < 2**63: raise ValueError("invalid native clock sample") return now diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index e75d801..1367e25 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -1726,6 +1726,8 @@ def test_authority_noninitial_native_claim_cannot_bypass_retry_verifier(tmp_path "proof_clock_unavailable", "proof_clock_regression", "proof_clock_replaced", + "proof_clock_descriptor", + "proof_clock_mode", ], ) @pytest.mark.parametrize("operation", ["claim", "report"]) @@ -1756,7 +1758,7 @@ def test_retry_verifier_is_rechecked_after_native_sql_staging( with sqlite3.connect(native_path) as connection: connection.execute("CREATE TABLE receipts (receipt BLOB NOT NULL)") connection.execute("INSERT INTO receipts VALUES (?)", (receipt,)) - state = {"staged": False, "checks": 0} + state = {"staged": False, "checks": 0, "late_callbacks": 0} lease_deadline = ( 10 if operation == "claim" else int(read("data/active-claim.json")["expires_at"]) ) @@ -1772,6 +1774,17 @@ def verify_receipts(evidence): host.commit_clock.set_native_sample(-1) elif damage == "proof_clock_replaced": host.commit_clock = NativeCommitClock.controlled(0) + elif damage == "proof_clock_descriptor": + sample = NativeCommitClock._sample.__get__(host.commit_clock, NativeCommitClock) + + def late_sample(_clock): + state["late_callbacks"] += 1 + host.handler._configured._provider.proof_verifier = None + return sample + + monkeypatch.setattr(NativeCommitClock, "_sample", property(late_sample)) + elif damage == "proof_clock_mode": + host.commit_clock._system = True if state["staged"] and damage == "unavailable": raise RuntimeError("native verifier unavailable after staging") with sqlite3.connect(native_path) as connection: @@ -1850,6 +1863,8 @@ def execute(self, sql, parameters=()): lambda: sqlite3.connect(host.path, isolation_level=None, factory=StagingConnection), ) arguments = {"expires_at": "10", "trusted_now": "0", "deduplication_evidence": proof} + if damage == "proof_clock_mode": + arguments["expires_at"] = read("data/active-claim.json")["expires_at"] def execute(): if operation == "claim": @@ -1871,13 +1886,21 @@ def execute(): else: code = ( "stale_attempt_fence" - if damage in {"proof_expiry", "proof_clock_unavailable", "proof_clock_regression"} + if damage + in { + "proof_expiry", + "proof_clock_unavailable", + "proof_clock_regression", + "proof_clock_descriptor", + "proof_clock_mode", + } else "host_capability_mismatch" ) with pytest.raises(EffectError, match=code): execute() assert state["staged"] assert host.snapshot(root) == before + assert state["late_callbacks"] == 0 @pytest.mark.parametrize("clock", ["expired", "unavailable", "invalid", "missing"]) From eeaaba1e554ae6903194ebd7270b6c15b8bd754d Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Tue, 6 Oct 2026 11:29:33 +0000 Subject: [PATCH 29/32] fix: preserve terminal effects and validate cancellation payloads --- .github/workflows/test.yml | 6 +- AGENTS.md | 4 +- README.md | 2 +- conformance/pins.py | 2 +- src/determa/state/effects.py | 35 ++++++- tests/test_effect_cancellation_parity.py | 116 +++++++++++++++++++++++ 6 files changed, 156 insertions(+), 9 deletions(-) create mode 100644 tests/test_effect_cancellation_parity.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 200ca4a..50b7aa8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,7 +53,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: dc84ed81ea36a5f2140181a97660477a14347ccc + ref: 090aa297389e416121c25ef29fb0a75671d83233 path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -88,7 +88,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: dc84ed81ea36a5f2140181a97660477a14347ccc + ref: 090aa297389e416121c25ef29fb0a75671d83233 path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -126,7 +126,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: dc84ed81ea36a5f2140181a97660477a14347ccc + ref: 090aa297389e416121c25ef29fb0a75671d83233 path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/AGENTS.md b/AGENTS.md index d68d6af..c056871 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,11 +12,11 @@ The implementation is conformant only when it passes the language-neutral suite. The implementation uses these immutable inputs: - specification: `77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0`; -- conformance: `384103d3709819052b3daaae6f89b531c5c3c146` (reviewed coordinated +- conformance: `090aa297389e416121c25ef29fb0a75671d83233` (reviewed coordinated draft #114; 99 format-1 core cases, 162 version-1 vectors, 142 durable-host vectors, 49 core inspection vectors, 47 extension negotiation vectors, 7 optional native-provider vectors, - 52 runtime-provider vectors, 55 committed-native-effect vectors, and 487 JSON artifacts). + 52 runtime-provider vectors, 62 committed-native-effect vectors, and 487 JSON artifacts). The package metadata is `0.3.0`. diff --git a/README.md b/README.md index 2bc7f86..cb346f2 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ commit `77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0`. Correctness is determined by 99 format-1 core cases, 162 version-1 vectors, 142 durable-host vectors, 49 core inspection vectors, 47 extension negotiation vectors, 7 native guard-provider inspection vectors, 52 runtime-provider vectors, and 487 manifest artifacts at -conformance commit `dc84ed81ea36a5f2140181a97660477a14347ccc`. The native +conformance commit `090aa297389e416121c25ef29fb0a75671d83233`. The native runtime-provider profile is optional and is exercised through its production adapter. The package metadata is `0.3.0`. Artifact and checkpoint schema version 1 is the diff --git a/conformance/pins.py b/conformance/pins.py index 9058ba6..1b8856f 100644 --- a/conformance/pins.py +++ b/conformance/pins.py @@ -4,7 +4,7 @@ from pathlib import Path -CONFORMANCE_COMMIT = "384103d3709819052b3daaae6f89b531c5c3c146" +CONFORMANCE_COMMIT = "090aa297389e416121c25ef29fb0a75671d83233" SPEC_COMMIT = "77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0" ROOT = Path(__file__).resolve().parent.parent diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index b2b4bd7..8a80849 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -2374,6 +2374,32 @@ def change(document: dict[str, Any], effect_id: str = effect_id) -> None: ) return self.snapshot(root) + def _validate_cancelled_payload( + self, checkpoint: dict[str, Any], record: dict[str, Any], payload: Any + ) -> None: + """Validate the original declaration without requiring current runtime liveness.""" + from .engine import _normalize_payload + from .wire import _origin_machine + + try: + target = _pinned_result_target(checkpoint, record, self.resolver) + delivery = _result_delivery_evidence(checkpoint, record, "cancelled", payload, target) + bundle, machine = _origin_machine( + self.resolver, record["target"]["runtime_incarnation"] + ) + declarations = dict(bundle.raw.get("events") or {}) + declarations.update(machine.raw.get("events") or {}) + declaration = declarations.get(_mapping(record, "cancelled")["event"]) + if not isinstance(declaration, dict) or declaration.get("direction") != "input": + raise EffectError("invalid_host_request") + logical = decoded_typed_value(delivery["envelope"]["payload"]) + if _normalize_payload(declaration, logical) is None: + raise EffectError("invalid_host_request") + except (ArtifactError, KeyError, TypeError, ValueError) as exc: + if isinstance(exc, EffectError): + raise + raise EffectError("invalid_host_request") from exc + def cancel(self, root: str, request: Mapping[str, Any]) -> dict[str, Any]: if ( not isinstance(request, Mapping) @@ -2442,7 +2468,11 @@ def change(document: dict[str, Any]) -> dict[str, Any]: ): raise EffectError("operation_id_conflict") return replay - if record["invocation_state"] == "unclaimed" and record["attempt_fence"] == "0": + self._validate_cancelled_payload(document["checkpoint"], record, request["payload"]) + if record["outcome"] is not None: + state = "too_late" + outcome = copy.deepcopy(record["outcome"]) + elif record["invocation_state"] == "unclaimed" and record["attempt_fence"] == "0": mapping = _mapping(record, "cancelled") self._result_delivery( document["checkpoint"], record, "cancelled", request["payload"] @@ -2476,7 +2506,8 @@ def change(document: dict[str, Any]) -> dict[str, Any]: "reason": request["reason"], "state": state, } - record["cancellation"] = cancellation + if state != "too_late": + record["cancellation"] = cancellation _bump(journal) response = { "status": "committed" if outcome else "reconciliation_required", diff --git a/tests/test_effect_cancellation_parity.py b/tests/test_effect_cancellation_parity.py new file mode 100644 index 0000000..d0ddd51 --- /dev/null +++ b/tests/test_effect_cancellation_parity.py @@ -0,0 +1,116 @@ +"""Late cancellation validates pinned payloads and preserves terminal evidence.""" + +import copy +import json +import sqlite3 + +import pytest + +from determa.state.effects import EffectError +from tests.test_committed_effects import ( + authority_claim_request, + authority_effect_fixture, + installed_test_handler, +) + + +def database_snapshot(host): + with sqlite3.connect(host.path) as connection: + return tuple(connection.iterdump()) + + +def invoked_host(tmp_path, state): + authority, host, scope, root, record = authority_effect_fixture(tmp_path) + command, invocation = authority_claim_request(authority, scope, root, record) + result = json.loads(authority.perform(command, invocation)) + assert result["status"] == "accepted" + claim = result["claim"] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + host.trusted_clock = lambda: "0" + calls = [] + + def sdk_call(*args): + calls.append(args) + return {"provider_reference": record["handler_reference"]["identifier"]} + + host = installed_test_handler(host, sdk_call) + context = { + "principal": claim["worker_principal"], + "scope": scope, + "epoch": "0", + "trusted_now": "0", + } + candidate = host.dispatch( + root, record["effect_id"], credential="native-test-credential", **context + ) + assert len(calls) == 1 + request = { + "effect_id": record["effect_id"], + "operation_token": record["operation_token"], + "attempt_fence": claim["attempt_fence"], + "outcome_kind": "succeeded", + "payload": ["map", [["provider_reference", ["string", candidate["provider_reference"]]]]], + } + if state == "outcome_recorded": + host._record_result(root, request, **context) + elif state == "result_admitted": + assert host.submit_result(root, request, **context)["status"] == "committed" + else: + assert state == "possible" + return host, root, record, calls + + +def cancellation(record, operation="late-cancel"): + return { + "operation_id": operation, + "effect_id": record["effect_id"], + "reason": "owner-stop", + "payload": ["map", []], + } + + +@pytest.mark.parametrize("state", ["outcome_recorded", "result_admitted"]) +def test_terminal_winner_returns_too_late_and_preserves_exact_record(tmp_path, state): + host, root, record, calls = invoked_host(tmp_path, state) + before = host.snapshot(root) + winning = copy.deepcopy(before["journal"]["effect_records"][0]) + request = cancellation(record) + reply = host.cancel(root, request) + assert reply["status"] == "committed" + assert reply["cancellation"]["state"] == "too_late" + assert reply["outcome"] == winning["outcome"] + assert reply["result_event_id"] == winning["result_event_id"] + after = host.snapshot(root) + assert after["checkpoint"] == before["checkpoint"] + assert after["journal"]["effect_records"][0] == winning + retained = database_snapshot(host) + assert host.cancel(root, request) == reply + assert database_snapshot(host) == retained + assert len(calls) == 1 + + +@pytest.mark.parametrize("state", ["possible", "outcome_recorded", "result_admitted"]) +def test_invalid_late_cancellation_payload_preserves_all_native_roles(tmp_path, state): + host, root, record, calls = invoked_host(tmp_path, state) + request = cancellation(record) + request["payload"] = ["map", [["undeclared", ["string", "value"]]]] + before = database_snapshot(host) + with pytest.raises(EffectError, match="invalid_host_request"): + host.cancel(root, request) + assert database_snapshot(host) == before + assert len(calls) == 1 + + +def test_new_late_cancel_preserves_original_prevented_start_evidence(tmp_path): + _, host, _, root, record = authority_effect_fixture(tmp_path) + first_request = cancellation(record, "cancel-first") + first = host.cancel(root, first_request) + assert first["cancellation"]["state"] == "prevented_start" + before = host.snapshot(root) + reply = host.cancel(root, cancellation(record, "cancel-late")) + assert reply["status"] == "committed" + assert reply["cancellation"]["state"] == "too_late" + assert reply["outcome"] == first["outcome"] + assert host.snapshot(root)["journal"]["effect_records"] == before["journal"]["effect_records"] From d7ff17397f22b532f6028f5c173621ed5c62ee7c Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Tue, 6 Oct 2026 11:53:50 +0000 Subject: [PATCH 30/32] fix: refuse standalone writes to native effects checkpoints --- src/determa/state/authority.py | 10 +- tests/test_native_checkpoint_ownership.py | 108 ++++++++++++++++++++++ 2 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 tests/test_native_checkpoint_ownership.py diff --git a/src/determa/state/authority.py b/src/determa/state/authority.py index d3ed6b7..2479668 100644 --- a/src/determa/state/authority.py +++ b/src/determa/state/authority.py @@ -1420,8 +1420,12 @@ def transaction(self, root_instance_id: str) -> Iterator[ExecutionStoreTransacti if root_instance_id not in ledger["roots"]: raise ExecutionStoreError("unauthorized_scope") try: + from .effects import _native_effect_documents + native_history = _native_checkpoint_history(ledger) - except ValueError as error: + native_effect_roots = set(_native_effect_documents(ledger)) + _native_checkpoints(connection, ledger) + except (ValueError, TypeError, KeyError, sqlite3.Error) as error: raise ExecutionStoreError("scope_fence_unproven") from error transaction = _SQLiteTransaction(connection, root_instance_id) previous = transaction.load() @@ -1430,6 +1434,10 @@ def transaction(self, root_instance_id: str) -> Iterator[ExecutionStoreTransacti yield transaction current = transaction.load() if current != previous: + # A native effects participant owns its checkpoint and journal + # together. Ordinary checkpoint-only writers cannot advance it. + if root_instance_id in native_effect_roots: + raise ExecutionStoreError("scope_fence_unproven") if current is None: raise ExecutionStoreError("invalid_execution_checkpoint") _, revision, digest = checkpoint_metadata(current) diff --git a/tests/test_native_checkpoint_ownership.py b/tests/test_native_checkpoint_ownership.py new file mode 100644 index 0000000..73e3e1c --- /dev/null +++ b/tests/test_native_checkpoint_ownership.py @@ -0,0 +1,108 @@ +"""Ordinary public checkpoint writes must not split native effects ownership.""" + +import json +import sqlite3 + +import pytest + +from determa.state import delivery_request_digest, portable_envelope +from determa.state.authority import AuthoritySQLiteExecutionStore +from determa.state.host import ExecutionHost, ExecutionHostError +from determa.state.stores.base import ExecutionStoreError +from determa.state.wire import canonical_bytes + +from .test_committed_effects import authority_effect_fixture + + +def database_snapshot(path): + with sqlite3.connect(path) as connection: + return "\n".join(connection.iterdump()) + + +def fixture(tmp_path, *, legacy_checkpoint=False): + authority, effects, scope, root, _record = authority_effect_fixture(tmp_path) + checkpoint = effects.snapshot(root)["checkpoint"] + store = AuthoritySQLiteExecutionStore(authority, scope, "owner", "0") + store.setup_schema() + if legacy_checkpoint: + # Earlier adapters permitted an equal second checkpoint representation. + # Reproduce that extant native state without using the now-refused writer. + with sqlite3.connect(authority.path) as connection: + connection.execute( + "INSERT INTO determa_execution_checkpoints VALUES (?,?,?,?)", + ( + root, + checkpoint["revision"], + checkpoint["execution_checkpoint_digest"], + canonical_bytes(checkpoint), + ), + ) + return authority, effects, root, checkpoint, store + + +@pytest.mark.parametrize("mutation", ["insert", "external_admission"]) +def test_ordinary_checkpoint_mutations_refuse_effects_owned_root(tmp_path, mutation): + authority, effects, root, checkpoint, store = fixture( + tmp_path, legacy_checkpoint=mutation == "external_admission" + ) + before = database_snapshot(authority.path) + initial = effects.snapshot(root) + with pytest.raises((ExecutionStoreError, ExecutionHostError), match="scope_fence_unproven"): + if mutation == "insert": + with store.transaction(root) as transaction: + transaction.insert(canonical_bytes(checkpoint)) + else: + runtime = checkpoint["root_record"]["aggregate_state"]["runtimes"][0] + envelope = portable_envelope( + "native_cancelled", + "external-admission", + {"root": {"root_instance_id": root, "root_runtime_id": runtime["runtime_id"]}}, + {}, + ) + delivery = { + "delivery_mode": "input", + "envelope": envelope, + "envelope_digest": delivery_request_digest(root, "input", envelope), + } + ExecutionHost(store, effects.resolver).admit_v1( + root, + [delivery], + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + assert database_snapshot(authority.path) == before + assert effects.snapshot(root) == initial + + +def test_ordinary_read_preserves_an_equal_legacy_checkpoint_and_native_history(tmp_path): + authority, effects, root, checkpoint, store = fixture(tmp_path, legacy_checkpoint=True) + before = database_snapshot(authority.path) + actual = ExecutionHost(store, effects.resolver).read_checkpoint(root) + assert actual is not None and actual.document == checkpoint + assert database_snapshot(authority.path) == before + + +@pytest.mark.parametrize("damage", ["missing_indexes", "empty_indexes"]) +def test_lost_native_indexes_cannot_reclassify_ordinary_writer_permission(tmp_path, damage): + authority, effects, root, checkpoint, store = fixture(tmp_path) + scope = effects.authority_scope + ledger = authority.inspect(scope) + for key in ("native_effect_roots", "native_effect_document_bytes"): + if damage == "missing_indexes": + del ledger[key] + else: + ledger[key] = [] + with authority._connect() as connection: + connection.execute( + "UPDATE determa_scope_authority SET ledger=? WHERE scope_identity=?", + (json.dumps(ledger), scope), + ) + connection.commit() + before = database_snapshot(authority.path) + entered = False + with pytest.raises(ExecutionStoreError, match="scope_fence_unproven"): + with store.transaction(root) as transaction: + entered = True + transaction.insert(canonical_bytes(checkpoint)) + assert not entered + assert database_snapshot(authority.path) == before From 39def1ac8268de55237f90e5cb8d7110fe27fc78 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Tue, 6 Oct 2026 12:04:04 +0000 Subject: [PATCH 31/32] fix: refuse checkpoint-only native SQLite access before execution --- src/determa/state/authority.py | 9 ++-- src/determa/state/stores/sqlite.py | 18 +++++++ tests/test_local_authority.py | 18 ++++++- tests/test_native_checkpoint_ownership.py | 61 +++++++++++++++++++++-- 4 files changed, 97 insertions(+), 9 deletions(-) diff --git a/src/determa/state/authority.py b/src/determa/state/authority.py index 2479668..3ab1e4a 100644 --- a/src/determa/state/authority.py +++ b/src/determa/state/authority.py @@ -1427,6 +1427,11 @@ def transaction(self, root_instance_id: str) -> Iterator[ExecutionStoreTransacti _native_checkpoints(connection, ledger) except (ValueError, TypeError, KeyError, sqlite3.Error) as error: raise ExecutionStoreError("scope_fence_unproven") from error + # Checkpoint-only execution cannot enter a joint checkpoint/journal + # participant, including reads through this write transaction API. + # Refuse before core actions or an application callback can run. + if root_instance_id in native_effect_roots: + raise ExecutionStoreError("scope_fence_unproven") transaction = _SQLiteTransaction(connection, root_instance_id) previous = transaction.load() if previous is not None and previous.decode("utf-8", "strict") not in native_history: @@ -1434,10 +1439,6 @@ def transaction(self, root_instance_id: str) -> Iterator[ExecutionStoreTransacti yield transaction current = transaction.load() if current != previous: - # A native effects participant owns its checkpoint and journal - # together. Ordinary checkpoint-only writers cannot advance it. - if root_instance_id in native_effect_roots: - raise ExecutionStoreError("scope_fence_unproven") if current is None: raise ExecutionStoreError("invalid_execution_checkpoint") _, revision, digest = checkpoint_metadata(current) diff --git a/src/determa/state/stores/sqlite.py b/src/determa/state/stores/sqlite.py index 0b6383d..842b9e1 100644 --- a/src/determa/state/stores/sqlite.py +++ b/src/determa/state/stores/sqlite.py @@ -41,6 +41,22 @@ _IMMUTABLE_MESSAGE = "execution_store_immutable" +def _refuse_native_owned_database(connection: sqlite3.Connection) -> None: + """The base adapter cannot prove native authority or joint journal ownership. + + Check permanent schema markers under the writer lock, before any caller or + core code runs. Even a partial native schema requires the native adapter. + """ + if ( + connection.execute( + "SELECT 1 FROM sqlite_master WHERE name COLLATE NOCASE IN (?, ?, ?) LIMIT 1", + ("determa_scope_authority", "determa_scope_allocations", "determa_committed_effects"), + ).fetchone() + is not None + ): + raise ExecutionStoreError("scope_fence_unproven") + + def _schema_tokens(source: str) -> list[str]: return re.findall(r"[a-z_][a-z0-9_]*|[(),]", source.lower()) @@ -443,6 +459,7 @@ def transaction( try: self._validate_schema(connection) connection.execute("BEGIN IMMEDIATE") + _refuse_native_owned_database(connection) transaction = _SQLiteTransaction(connection, root_instance_id) yield transaction connection.commit() @@ -532,6 +549,7 @@ def shared_transaction( try: self._validate_schema(connection) connection.execute("BEGIN IMMEDIATE") + _refuse_native_owned_database(connection) connection.set_authorizer(_application_authorizer) yield ( application, diff --git a/tests/test_local_authority.py b/tests/test_local_authority.py index 857b802..a244dda 100644 --- a/tests/test_local_authority.py +++ b/tests/test_local_authority.py @@ -271,7 +271,14 @@ def test_freeze_refuses_an_untracked_native_checkpoint_without_committing(tmp_pa response = authority.perform(_request("freeze_scope", "freeze-untracked", "0", {}), invocation) assert json.loads(response)["error_code"] == "scope_fence_unproven" assert authority.inspect("scope-1") == ledger - assert bare_host.read_checkpoint("foreign-root").canonical_bytes == before.canonical_bytes + with authority._connect() as connection: + actual = connection.execute( + "SELECT checkpoint FROM determa_execution_checkpoints WHERE root_instance_id=?", + ("foreign-root",), + ).fetchone() + assert actual is not None and bytes(actual[0]) == before.canonical_bytes + with pytest.raises(ExecutionStoreError, match="scope_fence_unproven"): + bare_host.read_checkpoint("foreign-root") @pytest.mark.parametrize( @@ -512,7 +519,14 @@ def test_guarded_write_cannot_adopt_a_preexisting_untracked_checkpoint(tmp_path) expected_checkpoint_digest=before.document["execution_checkpoint_digest"], ) assert authority.inspect("scope-1") == ledger - assert bare_host.read_checkpoint("root-1").canonical_bytes == before.canonical_bytes + with authority._connect() as connection: + actual = connection.execute( + "SELECT checkpoint FROM determa_execution_checkpoints WHERE root_instance_id=?", + ("root-1",), + ).fetchone() + assert actual is not None and bytes(actual[0]) == before.canonical_bytes + with pytest.raises(ExecutionStoreError, match="scope_fence_unproven"): + bare_host.read_checkpoint("root-1") @pytest.mark.parametrize("terminal", [False, True]) diff --git a/tests/test_native_checkpoint_ownership.py b/tests/test_native_checkpoint_ownership.py index 73e3e1c..91731cf 100644 --- a/tests/test_native_checkpoint_ownership.py +++ b/tests/test_native_checkpoint_ownership.py @@ -9,6 +9,7 @@ from determa.state.authority import AuthoritySQLiteExecutionStore from determa.state.host import ExecutionHost, ExecutionHostError from determa.state.stores.base import ExecutionStoreError +from determa.state.stores.sqlite import SQLiteExecutionStore from determa.state.wire import canonical_bytes from .test_committed_effects import authority_effect_fixture @@ -74,14 +75,68 @@ def test_ordinary_checkpoint_mutations_refuse_effects_owned_root(tmp_path, mutat assert effects.snapshot(root) == initial -def test_ordinary_read_preserves_an_equal_legacy_checkpoint_and_native_history(tmp_path): +def test_native_owned_transaction_refuses_before_ordinary_core_or_callback(tmp_path): authority, effects, root, checkpoint, store = fixture(tmp_path, legacy_checkpoint=True) before = database_snapshot(authority.path) - actual = ExecutionHost(store, effects.resolver).read_checkpoint(root) - assert actual is not None and actual.document == checkpoint + entered = False + with pytest.raises(ExecutionStoreError, match="scope_fence_unproven"): + with store.transaction(root): + entered = True + assert not entered + assert effects.snapshot(root)["checkpoint"] == checkpoint assert database_snapshot(authority.path) == before +@pytest.mark.parametrize("shared", [False, True]) +def test_base_sqlite_adapter_cannot_enter_authority_owned_transaction(tmp_path, shared): + authority, effects, root, checkpoint, _store = fixture(tmp_path, legacy_checkpoint=True) + store = SQLiteExecutionStore(authority.path, shared_application_transactions=shared) + before = database_snapshot(authority.path) + entered = False + with pytest.raises(ExecutionStoreError, match="scope_fence_unproven"): + if shared: + with store.shared_transaction(root) as (application, transaction): + entered = True + application.execute("CREATE TABLE application_side_effect (value TEXT)") + assert transaction.load() is not None + else: + with store.transaction(root) as transaction: + entered = True + assert transaction.load() is not None + assert not entered + assert database_snapshot(authority.path) == before + assert effects.snapshot(root)["checkpoint"] == checkpoint + + +@pytest.mark.parametrize( + "marker", + [ + "determa_scope_authority", + "determa_scope_allocations", + "determa_committed_effects", + "DETERMA_SCOPE_AUTHORITY", + "Determa_Scope_Allocations", + "Determa_Committed_Effects", + ], +) +@pytest.mark.parametrize("shared", [False, True]) +def test_partial_native_ownership_markers_fail_closed(tmp_path, marker, shared): + store = SQLiteExecutionStore( + tmp_path / "partial.sqlite", shared_application_transactions=shared + ) + store.setup_schema() + with sqlite3.connect(store.path) as connection: + connection.execute(f"CREATE TABLE {marker} (damaged TEXT)") + before = database_snapshot(store.path) + entered = False + with pytest.raises(ExecutionStoreError, match="scope_fence_unproven"): + context = store.shared_transaction("root") if shared else store.transaction("root") + with context: + entered = True + assert not entered + assert database_snapshot(store.path) == before + + @pytest.mark.parametrize("damage", ["missing_indexes", "empty_indexes"]) def test_lost_native_indexes_cannot_reclassify_ordinary_writer_permission(tmp_path, damage): authority, effects, root, checkpoint, store = fixture(tmp_path) From 76d71920eadee9da4972b7a0c2a4d7ca7c480a09 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Tue, 6 Oct 2026 14:25:26 +0000 Subject: [PATCH 32/32] Record reviewed prerequisites for native effect creation --- reviews/native-effect-creation-v030.md | 96 ++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 reviews/native-effect-creation-v030.md diff --git a/reviews/native-effect-creation-v030.md b/reviews/native-effect-creation-v030.md new file mode 100644 index 0000000..c55daca --- /dev/null +++ b/reviews/native-effect-creation-v030.md @@ -0,0 +1,96 @@ +# Native effect creation: bounded acceptance plan + +Status: first independent review requires shared creation/journal tests before +implementation. Python implementation has not started. + +This owner-local optional-host slice adds atomic creation to +`SQLiteCommittedEffectHost`. It does not change the portable core, add event +admission, implement a public transport endpoint, or start example integration. +Version 0.3.0 remains unreleased; schemas stay version 1 and machines `format: 1`. + +## Fixed inputs and prior evidence + +- Specification: `77c0a2e60cd0771a6d44ae170a079ddd51d7d9f0`, SPEC + §§17.3, 18 and 19.1–19.2, 19.5. +- Conformance: `090aa297389e416121c25ef29fb0a75671d83233`, reviewed draft + #114. The committed-native-effects generator exercises an existing admitted + event producing an intent; its 62 vectors do not prove native creation. +- Python baseline: `39def1ac8268de55237f90e5cb8d7110fe27fc78`. The + reviewed ownership boundary prevents ordinary adapters from bypassing native + ownership. `seed` restores existing artifacts; it is not atomic core creation. +- Rust baseline: `cc24d28d4f88790e1bdc046dad01867110dfc2be`. + `tests/native_handler.rs` already exercises actual creation, fresh trusted + source selection, retained replay, changed identity, and SIGKILL fate. Credit + that reviewed evidence; do not rewrite the existing implementation merely to + match Python's internal layout. + +## Contract and executable coverage + +| Obligation | Existing contract/tests | Required Python evidence | +|---|---|---| +| Actual core initialization produces revision 0, creation receipt and every initial external intent | §17.3; portable checkpoint creation cases | Real format-1 initialization with multiple effects; no hand-assembled candidate checkpoint | +| Checkpoint, all pinned effect records, exact original request/first response and native authority role evidence commit together | §§18, 19.1–19.2 | One SQLite transaction; reopened exact bytes and matching root/history inventory | +| Equal creation replay survives later checkpoint changes and unavailable executable source without a core/provider call | §§17.3, 19.2 | Retain original response; check request equality before resolver activation, route lookup or CAS; current authority authorization still applies | +| Changed creation ID, definition or typed bindings cannot reuse an existing root | §17.3 | Exact conflict, unchanged checkpoint/journal/authority bytes and zero core/provider calls | +| Trusted configured executable source and exact pinned route are checked before core and again at commit | §19.2; existing route-change vectors | Refuse untrusted/mismatched source, invalid mapping or changed route generation; post-staging source/route loss rolls back all native rows | +| Every initial intent has its own exact effect ID/digest, token, handler/destination/mapping and runtime incarnation | §§19.1–19.2 | Compare actual initial emissions to all journal records; support empty initial outbox; reject unsupported token/target combinations before commit | +| No hidden bypass through a competing helper-only or ordinary checkpoint row | §§18, 19.2; completed ownership tests | Concurrent root creation serializes; pre-existing ownership is rejected or replayed through retained native evidence, never overwritten | +| Process death preserves one atomic fate | §19.5 | Real child SIGKILL after SQL staging before commit, and after commit before response; reopen, compare bytes, retry exactly once without dispatch | + +The existing shared artifacts specify portable creation and journal semantics +separately; they do not execute their coupling at initialization. Add a shared +executable native-creation case before Python implementation: input is the actual +trusted definition, machine/root/creation identities, typed bindings and configured +route; observed output contains the exact actual creation checkpoint, receipt, +all initial external intents and journal pins, and retained first response. The +runner forwards inputs only. Independent semantic/hash checks must validate the +coupling, rather than accepting a copied expected result. + +The additional acceptance cases must distinguish committed faulted initialization +at revision 0 from rejection before any aggregate exists. A rejection reserves no +root and permits a later valid creation. Preserve creation receipt sequence 0, +initialization lifecycle receipts in defined order, and internal mailbox emissions, +in addition to all initial external effects. Creation identity uses §17.3 normalized +typed-binding digest semantics, including numeric types and defaults, rather than +ad hoc raw request equality. Equal replay and conflict also apply after tombstoning. + +Machine-visible operation tokens must originate in declared bindings/variables and +emitted data. A host-only token may instead use the §19.1 derivation, without +claiming that the machine already knows it. Do not interchange a separate optional +owner-local operation identity with the creation identity or imply §25 transport +semantics. The exact owner-local response contract must be fixed by these cases. + +Fresh creation requires active authorized native authority and permanent allocation +evidence. It must not activate seed/imported artifacts, repair history gaps, or +overwrite pre-existing checkpoint/helper ownership. Recheck source and authorization +at the actual commit boundary. Creation makes zero SDK handler calls; independently +count zero core/provider calls on replay and conflict. + +Language-native transaction, source-race and SIGKILL tests remain additional +required implementation evidence. The existing approved prose suffices; no new +portable behavior, grammar or schema version is proposed. + +## Ordered checkpoints and reviews + +1. Independently review this contract/test mapping before source changes. +2. Add and independently review the shared executable creation/journal cases, + deterministic generator, semantic validator and input-only runner. Push this + conformance checkpoint before Python source changes. Then add meaningful + failing tests against the unchanged Python baseline, using the + actual configured authority and handler fixtures. Record RED separately from + setup failures. Commit and ordinary-push the explicitly unfinished checkpoint; + verify the remote SHA. +3. Implement creation only. Freeze each meaningful checkpoint for focused checks + and independent review; commit and ordinary-push corrections. Keep PR #111 draft. +4. At the final clean Python SHA run static checks, full unit/conformance tests, + actual configured 62-vector effects/authority gate, actual PostgreSQL checks, + and independent review. Parent gates are not current-head evidence. +5. Review Rust parity against that acceptance set. Reuse exact-head existing + evidence where unchanged; run focused additional tests for any new case, and + review/gate any correction separately. No parallel Cargo runs. +6. Mark creation complete in the cross-repository audit only after these proofs. + Choose event admission as its own next slice. Examples and publication remain + blocked on their wider dependencies. + +No force pushes, main writes, tags, package publication, release dispatch or +deployment are part of these checkpoints.