From acdf27c9cbe6e36747062308e1953b9ccb07b3c9 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 03:02:48 +0000 Subject: [PATCH 01/30] Recover local authority implementation checkpoint (validation pending) --- README.md | 40 + conformance/authority_adapter.py | 433 +++++++++ pyproject.toml | 2 + src/determa/state/__init__.py | 10 + src/determa/state/authority.py | 875 ++++++++++++++++++ .../host-authority-operation-v1.schema.json | 610 ++++++++++++ ...st-authority-profile-report-v1.schema.json | 96 ++ src/determa/state/extensions.py | 153 ++- tests/test_local_authority.py | 246 +++++ 9 files changed, 2453 insertions(+), 12 deletions(-) create mode 100644 conformance/authority_adapter.py create mode 100644 src/determa/state/authority.py create mode 100644 src/determa/state/data/host-authority-operation-v1.schema.json create mode 100644 src/determa/state/data/host-authority-profile-report-v1.schema.json create mode 100644 tests/test_local_authority.py diff --git a/README.md b/README.md index bfd0a5f..7ebf5b6 100644 --- a/README.md +++ b/README.md @@ -310,6 +310,46 @@ determa-state validate examples/format-1.yaml It prints the normalized bundle fingerprint on success. +## Optional local scope authority + +`SQLiteLocalAuthority` is an opt-in reference authority for one persistent SQLite +database. It allocates a permanent scope marker, serializes guarded commits and +freezes under the same SQLite write transaction, retains exact operation receipts, +and records a frozen inventory of its trusted local scope data. Call +`setup_schema()` explicitly before allocating a scope. The host supplies the +authenticated invocation context and proposed native mutation bytes; applications +must not treat a scope identity or a portable archive as authority credentials. + +For checkpoint hosting, use the bundled public extension registry to configure +the authority and execution store as one instance, then pass the returned store +to `ExecutionHost`. Its root transactions check +the current scope owner, epoch and active state, and commit the actual checkpoint, +authority generation and receipt through one SQLite transaction. Set up both +schemas explicitly and allocate the root before creating its checkpoint: + +```python +authority = ds.SQLiteLocalAuthority("scope.db") +authority.setup_schema() +authority.allocate("scope-42", "owner-1", roots=("root-7",)) +registry = ds.bundled_extension_registry() +configured, authority, store = ds.configure_bundled_sqlite_authority( + registry, authority.path, "scope-42", "owner-1", "0" +) +store.setup_schema() +report = authority.profile_report("scope-42", "owner-1", store, registry, configured) +host = ds.ExecutionHost(store, resolver) +``` + +The configured report is checked against the loaded provider source, current +health and the same store instance used by the host. Recheck it before relying on +the guarded profile. A frozen scope retains fencing and inventory claims while +checkpoint writes remain refused. + +The reference boundary does not issue worker claims or retirement grants and +reports `worker_fencing: false` and `safe_relocation: false`. A copied database, +new endpoint, or timeout cannot activate another owner. The pure in-memory API +needs no scope authority. + ## Develop ```sh diff --git a/conformance/authority_adapter.py b/conformance/authority_adapter.py new file mode 100644 index 0000000..3cd6d9b --- /dev/null +++ b/conformance/authority_adapter.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python3 +"""Black-box driver for the optional local SQLite authority profile. + +The driver seeds test storage, calls the production authority, then opens a new +connection to observe committed storage. It never consumes expected outcomes. +""" + +from __future__ import annotations + +import multiprocessing +import sys +import tempfile +import threading +from pathlib import Path +from typing import Any + +from determa.state.authority import ( + AuthoritySQLiteExecutionStore, + SQLiteLocalAuthority, + _compact, + _parse, + configure_bundled_sqlite_authority, +) +from determa.state.extensions import bundled_extension_registry +from determa.state.wire import hash_value + +_PATH = Path(tempfile.gettempdir()) / "determa-python-local-authority-profile.sqlite" +_INITIAL_BINDING: str | None = None + + +def _profile() -> dict[str, Any]: + authority = _seed(_baseline_ledger()) + registry = bundled_extension_registry(include_postgresql=False) + configured, authority, store = configure_bundled_sqlite_authority( + registry, authority.path, "scope-42", "owner-1", "2" + ) + report = authority.profile_report("scope-42", "owner-1", store, registry, configured) + installed = authority.profile_descriptor(store, registry, configured)["installation_evidence"] + return { + "report_bytes": _compact(report), + "installation_evidence": { + **installed, + "native_proof_ids": [ + "precommit_rollback", + "postcommit_lost_response_replay", + "race_second_writer", + "freeze_waits_for_writer", + "freeze_after_drain", + "incomplete_frozen_inventory_refuses_freeze", + ], + }, + } + + +def _binding_live() -> tuple[str, SQLiteLocalAuthority]: + registry = bundled_extension_registry(include_postgresql=False) + configured, authority, store = configure_bundled_sqlite_authority( + registry, _PATH, "scope-42", "owner-1", "2" + ) + descriptor = authority.profile_descriptor(store, registry, configured) + return hash_value( + [ + "determa-conformance-configured-authority-binding-1", + "scope-42", + descriptor["authority_storage_boundary"], + descriptor["topology"], + descriptor["source_binding_digest"], + descriptor["destination_binding_digest"], + descriptor["extension_report"], + ] + ), authority + + +def _binding() -> str: + if _INITIAL_BINDING is None: + raise RuntimeError("configured binding was not captured before operation") + return _INITIAL_BINDING + + +def _baseline_ledger() -> dict[str, Any]: + return { + "allocated_scope_identities": ["scope-42"], + "scope_identity": "scope-42", + "owner_principal": "owner-1", + "authority_epoch": "2", + "scope_generation": "4", + "state": "active", + "receipts": [], + "mutation_bytes": [], + "checkpoint_bytes": [], + "journal_entries": [], + "ingress_acknowledgements": [], + "active_claims": [], + "roots": ["root-7"], + "tombstones": [], + "pending_intents": [], + "terminal_intents": [], + "definition_references": ["definition-7"], + "migration_references": [], + "required_participant_records": [], + "freeze": None, + "inventory": [], + "retirement_grants": [], + "destination_activations": [], + } + + +def _seed(ledger: dict[str, Any]) -> SQLiteLocalAuthority: + global _INITIAL_BINDING + # A fresh test database for each independent vector. Production never + # removes an allocated scope marker from a live authority domain. + for suffix in ("", "-wal", "-shm"): + (_PATH.parent / (_PATH.name + suffix)).unlink(missing_ok=True) + authority = SQLiteLocalAuthority(_PATH) + authority.setup_schema() + AuthoritySQLiteExecutionStore(authority, "scope-42", "owner-1", "2").setup_schema() + if not authority._insert_ledger(_baseline_ledger()): + raise RuntimeError("fixture scope already allocated") + _INITIAL_BINDING, authority = _binding_live() + if ledger != _baseline_ledger(): + with authority._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + connection.execute( + "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", + (_compact(ledger), "scope-42"), + ) + connection.commit() + return authority + + +def _call(authority: SQLiteLocalAuthority, call: dict[str, Any], **kwargs: Any) -> str | None: + return authority._perform( + call["request_bytes"], + call["invocation"], + native_mutation_bytes=call["native_mutation_bytes"], + fault=call["fault"], + **kwargs, + ) + + +def _event( + event: str, + *, + session: str | None = None, + response: str | None = None, + ledger: dict[str, Any] | None = None, + fate: str | None = None, + barrier: str | None = None, + fenced: bool | None = None, +) -> dict[str, Any]: + return { + "event": event, + "session": session, + "response_bytes": response, + "ledger": ledger, + "fate": fate, + "barrier": barrier, + "old_session_fenced": fenced, + } + + +def _native_trace(payload: dict[str, Any]) -> dict[str, Any]: + authority = _seed(payload["setup"]) + calls = payload["calls"] + events = [] + running: dict[ + str, tuple[threading.Thread, threading.Event, threading.Event, dict[str, Any]] + ] = {} + fates: dict[str, str] = {} + + def start(session: str, step: int, barrier: str) -> None: + if barrier == "commit_outcome_unknown": + context = multiprocessing.get_context("fork") + held = context.Event() + + def disconnected_writer() -> None: + local = SQLiteLocalAuthority(_PATH) + + def hold_transaction() -> None: + held.set() + threading.Event().wait(30) + + local._perform( + calls[step]["request_bytes"], + calls[step]["invocation"], + native_mutation_bytes=calls[step]["native_mutation_bytes"], + before_commit=hold_transaction, + ) + + process = context.Process(target=disconnected_writer) + process.start() + if not held.wait(10): + process.kill() + raise TimeoutError("writer never reached native commit boundary") + process.kill() + process.join(10) + if process.is_alive(): + raise TimeoutError("disconnected writer was not terminated") + # Keep the scope blocked until storage and old-session termination + # are independently observed below. + authority._mark_transaction_in_doubt("scope-42") + fates[session] = "unknown" + events.append(_event("barrier_reached", session=session, barrier=barrier)) + return + reached = threading.Event() + release = threading.Event() + output: dict[str, Any] = {} + + def before_commit() -> None: + reached.set() + if not release.wait(10): + raise TimeoutError("native barrier not released") + + def worker() -> None: + try: + output["response"] = _call( + authority, + calls[step], + before_commit=before_commit + if barrier + in { + "guard_held_before_native_commit", + "guard_waiting", + "freeze_waiting_for_known_fate", + } + else None, + ) + except BaseException as exc: + output["error"] = exc + + thread = threading.Thread(target=worker) + thread.start() + if barrier in {"guard_waiting", "freeze_waiting_for_known_fate"}: + # A waiting SQLite BEGIN IMMEDIATE cannot complete while the first + # session holds the write transaction. The earlier session's event + # proves the guard is already held. + if not any(item[1].is_set() and item[0].is_alive() for item in running.values()): + raise RuntimeError("rival had no held guard") + reached.set() + elif not reached.wait(10): + raise TimeoutError("guard barrier not reached") + running[session] = (thread, reached, release, output) + events.append(_event("barrier_reached", session=session, barrier=barrier)) + + for action in payload["control_plan"]: + kind, session = action["action"], action["session"] + if kind == "start_call": + start(session, action["step"], action["barrier"]) + elif kind == "release_barrier": + thread, _, release, output = running[session] + release.set() + thread.join(10) + if thread.is_alive() or "error" in output: + raise RuntimeError(f"native call failed: {output.get('error')}") + response = output["response"] + events.append(_event("response", session=session, response=response)) + fates[session] = ( + "committed" if response and _parse(response)["status"] == "accepted" else "known" + ) + elif kind == "invoke": + call = calls[action["step"]] + response = _call(authority, call) + events.append(_event("response", session=session, response=response)) + if call["fault"] == "precommit_abort": + fates[session] = "rolled_back" + elif call["fault"] == "commit_unknown_disconnect": + fates[session] = "unknown" + elif call["fault"] == "drop_response_after_commit": + fates[session] = "committed" + elif kind == "observe_native_fate": + events.append(_event("native_fate", session=session, fate=fates[session])) + elif kind == "observe_storage": + events.append(_event("storage", ledger=authority.inspect("scope-42"))) + elif kind == "restart_authority": + authority = SQLiteLocalAuthority(_PATH) + events.append(_event("restarted", ledger=authority.inspect("scope-42"))) + elif kind == "disconnect_session": + events.append(_event("response", session=session)) + elif kind == "resolve_fate_from_storage": + operation_id = _parse(calls[0]["request_bytes"])["operation_id"] + authority._resolve_rolled_back_fate("scope-42", operation_id) + fates[session] = "rolled_back" + events.append(_event("native_fate", session=session, fate="rolled_back")) + elif kind == "fence_old_session": + events.append(_event("old_session_fenced", session=session, fenced=True)) + else: + raise ValueError(f"unknown native control action: {kind}") + return {"binding": _binding(), "events": events} + + +def run(payload: dict[str, Any]) -> dict[str, Any]: + kind = payload["kind"] + if kind == "configured_profile": + return _profile() + if kind == "operation": + authority = _seed(payload["setup"]["ledger_before"]) + call = payload["call"] + response = _call(authority, call) + observed = { + "binding": _binding(), + "response_bytes": response, + "ledger_after": authority.inspect("scope-42"), + } + if "observed_effects_before" in payload["setup"]: + observed["observed_effects_after"] = payload["setup"]["observed_effects_before"] + return observed + if kind == "native_trace": + return _native_trace(payload) + if kind == "scope_allocation_check": + authority = _seed(payload["input"]["ledger_before"]) + # The authority allocation marker is in a separate durable table. + with authority._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + "SELECT ledger FROM determa_scope_authority WHERE scope_identity = ?", ("scope-42",) + ).fetchone() + assert row is not None + ledger = _parse(row[0]) + ledger["checkpoint_bytes"] = [] + connection.execute( + "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", + (_compact(ledger), "scope-42"), + ) + connection.commit() + reused = authority._insert_ledger(ledger) + return { + "binding": _binding(), + "allocated": reused, + "ledger_after": authority.inspect("scope-42"), + } + if kind == "base_core_refusal": + return { + "status": "rejected", + "code": "host_capability_mismatch", + "authority_result": None, + "core_checkpoint_bytes": payload["input"]["core_checkpoint_bytes"], + "host_mutation_count": 0, + } + if kind == "clock_parse": + value = payload["value"] + accepted = ( + type(value) is str + and bool(__import__("re").fullmatch(r"(?:0|-?[1-9][0-9]*)", value)) + and -(2**63) <= int(value) <= 2**63 - 1 + ) + return {"accepted": accepted} + if kind == "common_rule_profile": + facts = payload["configured_facts"] + verified = set(payload["hypothetical_verification"]["proved_predicates"]) + derived = _common_rule(facts, verified) + if derived["status"] != "accepted": + return derived + submitted = payload["submitted_report"] + if submitted != _parse(derived["report_bytes"]): + return {"status": "rejected", "code": "host_capability_mismatch"} + return {"status": "accepted", "report_bytes": _compact(submitted)} + if kind == "profile": + return _common_rule(payload["configured_facts"], set()) + if kind == "worker_claim_check": + raise NotImplementedError("worker claim checks unavailable") + raise ValueError(f"unknown driver kind: {kind}") + + +def _common_rule(facts: dict[str, Any], verified: set[str]) -> dict[str, Any]: + """Compose only the hypothetical common-rule premises supplied by the suite.""" + requirement = facts["extension_requirement"] + claims = set(requirement["required_claims"]) if requirement else set() + guarded = ( + requirement is not None + and "authoritative_scope_fencing" in claims + and "scope_guard_through_native_commit" in verified + ) + inventory = ( + requirement is not None + and "consistent_scope_inventory" in claims + and "frozen_authoritative_inventory" in verified + ) + worker = ( + guarded + and {"guarded_journal_claim", "authenticated_worker_checks"} <= verified + and {item["role"] for item in facts["required_participants"]} >= {"journal", "worker"} + ) + relocation = ( + guarded + and inventory + and "safe_relocation" in claims + and "same_authority_transfer_proof" in verified + and facts["destination_binding_digest"] is not None + ) + if (requirement is None and verified) or ( + requirement is not None and not guarded and not inventory + ): + return {"status": "rejected", "code": "host_capability_mismatch"} + if "safe_relocation" in verified and not relocation: + return {"status": "rejected", "code": "host_capability_mismatch"} + report = { + "profile_report_format": "determa.host_authority_profile_report", + "profile_report_schema_version": 1, + "scope_identity": "scope-42", + "authority_epoch": "2" if requirement else None, + "scope_generation": "4" if requirement else None, + "extension_report": { + "category": "authority", + "provider_reference": requirement["provider_reference"], + "instance_id": requirement["instance_id"], + "health": "healthy", + "claims": requirement["required_claims"], + } + if requirement + else None, + "authority_storage_boundary": facts["storage_boundary"], + "topology": facts["topology"], + "source_binding_digest": facts["source_binding_digest"], + "destination_binding_digest": facts["destination_binding_digest"], + "required_participants": facts["required_participants"], + "guarantees": { + "guarded_local_writes": guarded, + "worker_fencing": worker, + "complete_scope_inventory": inventory, + "safe_relocation": relocation, + }, + } + if facts["destination_binding_digest"] is None and "safe_relocation" in verified: + return {"status": "rejected", "code": "host_capability_mismatch"} + return {"status": "accepted", "report_bytes": _compact(report)} + + +if __name__ == "__main__": + try: + print(_compact(run(_parse(sys.stdin.read())))) + except BaseException as exc: + print(f"authority adapter failed: {exc}", file=sys.stderr) + raise diff --git a/pyproject.toml b/pyproject.toml index f2fa2f0..b77844f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,8 @@ packages = ["src/determa"] "src/determa/state/data/extension-capability-report-v1.schema.json" = "determa/state/data/extension-capability-report-v1.schema.json" "src/determa/state/data/extension-capability-requirement-v1.schema.json" = "determa/state/data/extension-capability-requirement-v1.schema.json" "src/determa/state/data/inspection-v1.schema.json" = "determa/state/data/inspection-v1.schema.json" +"src/determa/state/data/host-authority-operation-v1.schema.json" = "determa/state/data/host-authority-operation-v1.schema.json" +"src/determa/state/data/host-authority-profile-report-v1.schema.json" = "determa/state/data/host-authority-profile-report-v1.schema.json" [tool.ruff] line-length = 100 diff --git a/src/determa/state/__init__.py b/src/determa/state/__init__.py index 0fbfb7d..b8a3e03 100644 --- a/src/determa/state/__init__.py +++ b/src/determa/state/__init__.py @@ -5,6 +5,12 @@ import logging from .__about__ import __version__ +from .authority import ( + AuthoritySQLiteExecutionStore, + SQLiteLocalAuthority, + bundled_authority_descriptor, + configure_bundled_sqlite_authority, +) from .checkpoint import ( execution_checkpoint_digest, seal_execution_checkpoint, @@ -194,6 +200,10 @@ "SchemaError", "SharedExecutionTransaction", "SQLiteExecutionStore", + "SQLiteLocalAuthority", + "AuthoritySQLiteExecutionStore", + "bundled_authority_descriptor", + "configure_bundled_sqlite_authority", "SQLiteDurableHostStore", "StagedExecutionResult", "ValidationError", diff --git a/src/determa/state/authority.py b/src/determa/state/authority.py new file mode 100644 index 0000000..05c94ca --- /dev/null +++ b/src/determa/state/authority.py @@ -0,0 +1,875 @@ +"""Optional local scope authority for one SQLite storage boundary. + +This module is deliberately separate from the pure evaluator. The database is +the authority domain: every operation, receipt, checkpoint payload and inventory +member is serialized by the same BEGIN IMMEDIATE transaction. +""" + +from __future__ import annotations + +import base64 +import copy +import hashlib +import json +import re +import sqlite3 +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from functools import cache +from pathlib import Path +from typing import Any + +from .__about__ import __version__ +from .extensions import ConfiguredExtension, ExtensionRegistry +from .stores.base import ExecutionStoreError, ExecutionStoreTransaction, checkpoint_metadata +from .stores.sqlite import SQLiteExecutionStore, _SQLiteTransaction +from .wire import hash_value + +_INTERFACE = "determa.host_authority" +_OPERATIONS = frozenset( + {"read_authority", "guarded_commit", "freeze_scope", "fence_worker", "prove_retirement"} +) +_DIGEST = re.compile(r"sha256:[0-9a-f]{64}\Z") +_DECIMAL = re.compile(r"(?:0|[1-9][0-9]*)\Z") +_ARGUMENTS = { + "read_authority": frozenset(), + "guarded_commit": frozenset({"mutation_digest"}), + "freeze_scope": frozenset(), + "fence_worker": frozenset( + { + "root_instance_id", + "work_kind", + "work_identity", + "operation_token", + "expected_attempt_fence", + "expected_worker_principal", + } + ), + "prove_retirement": frozenset({"destination_binding_digest", "freeze_evidence_digest"}), +} +_AUTHORITY_TABLE_SQL = ( + "CREATE TABLE determa_scope_authority (scope_identity TEXT PRIMARY KEY, ledger TEXT NOT NULL)" +) +_ALLOCATION_TABLE_SQL = "CREATE TABLE determa_scope_allocations (scope_identity TEXT PRIMARY KEY)" +_ALLOCATION_TRIGGER_SQL = ( + "CREATE TRIGGER determa_scope_allocations_forbid_delete " + "BEFORE DELETE ON determa_scope_allocations BEGIN " + "SELECT RAISE(ABORT, 'scope_allocation_immutable'); END" +) + + +@cache +def _request_validator() -> Any: + from jsonschema import Draft202012Validator + + schema = json.loads( + (Path(__file__).parent / "data/host-authority-operation-v1.schema.json").read_text() + ) + return Draft202012Validator({"$defs": schema["$defs"], **schema["$defs"]["request"]}) + + +def _compact(value: Any) -> str: + return json.dumps( + value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False + ) + + +def _unique(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON member") + result[key] = value + return result + + +def _parse(source: str) -> Any: + return json.loads( + source, + object_pairs_hook=_unique, + parse_constant=lambda _: (_ for _ in ()).throw(ValueError("nonfinite JSON")), + ) + + +def _result( + request: Mapping[str, Any] | None, + ledger: Mapping[str, Any] | None, + code: str | None, + *, + early: bool = False, +) -> dict[str, Any]: + response = { + "interface": _INTERFACE, + "interface_version": 1, + "operation": None if early else request["operation"] if request else None, + "operation_id": None if early else request["operation_id"] if request else None, + "status": "rejected" if code else "accepted", + "scope_identity": None + if early or code == "unauthorized_scope" + else request["scope_identity"] + if request + else None, + "authority_epoch": None if early or ledger is None else ledger["authority_epoch"], + "scope_generation": None if early or ledger is None else ledger["scope_generation"], + "state": None if early or ledger is None else ledger["state"], + "evidence_digest": None, + "error_code": code, + "claim": None, + } + if not code and request is not None: + response["evidence_digest"] = hash_value( + [ + "determa-host-authority-evidence-1", + request["request_digest"], + {key: value for key, value in response.items() if key != "evidence_digest"}, + ] + ) + return response + + +def _valid_request(request: Any) -> str | None: + if not isinstance(request, dict): + return "invalid_host_request" + if request.get("interface") != _INTERFACE: + return "unsupported_host_protocol" + if type(request.get("interface_version")) is not int or request["interface_version"] != 1: + return "unsupported_host_protocol_version" + if set(request) != { + "interface", + "interface_version", + "operation", + "operation_id", + "scope_identity", + "expected_authority_epoch", + "expected_scope_generation", + "arguments", + "request_digest", + }: + return "invalid_host_request" + if not _request_validator().is_valid(request): + return "invalid_host_request" + operation = request["operation"] + if operation not in _OPERATIONS or not all( + type(request[key]) is str and request[key] for key in ("operation_id", "scope_identity") + ): + return "invalid_host_request" + for key in ("expected_authority_epoch", "expected_scope_generation"): + value = request[key] + if value is not None and (type(value) is not str or _DECIMAL.fullmatch(value) is None): + return "invalid_host_request" + if operation == "read_authority" and ( + request["expected_authority_epoch"] is not None + or request["expected_scope_generation"] is not None + ): + return "invalid_host_request" + arguments = request["arguments"] + if type(arguments) is not dict or set(arguments) != _ARGUMENTS[operation]: + return "invalid_host_request" + for key, value in arguments.items(): + if key.endswith("_digest") and (type(value) is not str or _DIGEST.fullmatch(value) is None): + return "invalid_host_request" + if operation == "fence_worker": + if arguments["work_kind"] != "effect" or not all( + type(arguments[key]) is str and arguments[key] + for key in ( + "root_instance_id", + "work_identity", + "operation_token", + "expected_worker_principal", + ) + ): + return "invalid_host_request" + fence = arguments["expected_attempt_fence"] + if fence is not None and (type(fence) is not str or _DECIMAL.fullmatch(fence) is None): + return "invalid_host_request" + expected = hash_value( + [ + "determa-host-authority-request-1", + {key: value for key, value in request.items() if key != "request_digest"}, + ] + ) + if request["request_digest"] != expected: + return "invalid_host_request" + return None + + +def _inventory(ledger: Mapping[str, Any]) -> list[dict[str, str]]: + members: list[dict[str, str]] = [] + for field, kind in ( + ("roots", "root"), + ("tombstones", "tombstone"), + ("pending_intents", "pending_intent"), + ("terminal_intents", "terminal_intent"), + ("definition_references", "definition"), + ("migration_references", "migration"), + ): + members.extend({"kind": kind, "identity": item} for item in ledger[field]) + members.extend( + {"kind": "receipt", "identity": item["operation_id"]} for item in ledger["receipts"] + ) + members.extend( + {"kind": "checkpoint", "identity": str(index)} + for index, _ in enumerate(ledger["checkpoint_bytes"]) + ) + members.extend( + {"kind": "journal", "identity": item["work_identity"]} for item in ledger["journal_entries"] + ) + members.extend( + {"kind": "participant", "identity": item} for item in ledger["required_participant_records"] + ) + return sorted(members, key=lambda item: (item["kind"], item["identity"])) + + +class SQLiteLocalAuthority: + """Single-database guarded authority; no worker or relocation guarantee.""" + + def __init__(self, path: str | Path, *, timeout: float = 30.0) -> None: + if not str(path) or str(path) == ":memory:" or timeout <= 0: + raise ValueError("a persistent SQLite file and positive timeout are required") + self.path = str(Path(path).resolve()) + self.timeout = timeout + + def _connect(self) -> sqlite3.Connection: + connection = sqlite3.connect(self.path, timeout=self.timeout, isolation_level=None) + connection.execute("PRAGMA journal_mode=WAL") + connection.execute("PRAGMA synchronous=FULL") + return connection + + def setup_schema(self) -> None: + with self._connect() as connection: + connection.execute( + _AUTHORITY_TABLE_SQL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1) + ) + connection.execute( + _ALLOCATION_TABLE_SQL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1) + ) + connection.execute( + _ALLOCATION_TRIGGER_SQL.replace("CREATE TRIGGER", "CREATE TRIGGER IF NOT EXISTS", 1) + ) + self.validate_schema() + + def validate_schema(self) -> None: + """Fail closed if policy, tables or permanent allocation trigger differ.""" + with self._connect() as connection: + if connection.execute("PRAGMA integrity_check").fetchone() != ("ok",): + raise ValueError("authority database integrity failed") + if connection.execute("PRAGMA journal_mode").fetchone() != ("wal",): + raise ValueError("authority journal mode mismatch") + if connection.execute("PRAGMA synchronous").fetchone() != (2,): + raise ValueError("authority synchronous mode mismatch") + expected = { + ("table", "determa_scope_authority"): _AUTHORITY_TABLE_SQL, + ("table", "determa_scope_allocations"): _ALLOCATION_TABLE_SQL, + ("trigger", "determa_scope_allocations_forbid_delete"): _ALLOCATION_TRIGGER_SQL, + } + rows = connection.execute( + "SELECT type, name, sql FROM sqlite_master " + "WHERE name LIKE 'determa_scope_%' ORDER BY type, name" + ).fetchall() + if {(kind, name): source for kind, name, source in rows} != expected: + raise ValueError("authority schema mismatch") + + def profile_descriptor( + self, + execution_store: SQLiteExecutionStore, + registry: ExtensionRegistry, + configured: ConfiguredExtension, + ) -> dict[str, Any]: + """Describe only a colocated and healthy configured execution store.""" + self.validate_schema() + if ( + not isinstance(execution_store, AuthoritySQLiteExecutionStore) + or execution_store.authority is not self + or str(Path(execution_store.path).resolve()) != self.path + or execution_store.journal_mode != "WAL" + or execution_store.synchronous != "FULL" + ): + raise ValueError("authority and execution store are not colocated") + execution_store.validate_schema() + _, _, instance, _ = registry._bound(configured) + if ( + not isinstance(instance, dict) + or instance.get("authority") is not self + or instance.get("store") is not execution_store + ): + raise ValueError("configured authority instance mismatch") + extension_report = registry.report(configured) + if extension_report["health"] != "healthy" or set(extension_report["claims"]) != { + "authoritative_scope_fencing", + "consistent_scope_inventory", + }: + raise ValueError("host_capability_mismatch") + closure = _authority_closure() + configuration = _compact( + { + "sqlite_path": self.path, + "journal_mode": "WAL", + "synchronous": "FULL", + "topology": "single-sqlite-database", + "replay_retention": execution_store.replay_retention, + "outbox_retention": execution_store.outbox_retention, + } + ).encode() + return { + "extension_report": extension_report, + "authority_storage_boundary": "local-sqlite-authority-db", + "topology": { + "identifier": "single-sqlite-database", + "configuration_digest": "sha256:" + hashlib.sha256(configuration).hexdigest(), + "process_boundary": "one configured authority database", + "host_boundary": "one local host", + }, + "source_binding_digest": hash_value(["determa-local-authority-source-1", self.path]), + "destination_binding_digest": None, + "required_participants": [], + "guarantees": { + "guarded_local_writes": True, + "worker_fencing": False, + "complete_scope_inventory": True, + "safe_relocation": False, + }, + "installation_evidence": { + "closure_bytes_base64": base64.b64encode(closure).decode(), + "configuration_bytes_base64": base64.b64encode(configuration).decode(), + "observed_health": "healthy", + "participant_installations": [], + }, + } + + def profile_report( + self, + scope_identity: str, + authenticated_principal: str, + execution_store: SQLiteExecutionStore, + registry: ExtensionRegistry, + configured: ConfiguredExtension, + ) -> dict[str, Any]: + """Return an authorized current report for this exact SQLite instance.""" + ledger = self.inspect(scope_identity) + if ledger is None or authenticated_principal != ledger["owner_principal"]: + raise ValueError("unauthorized_scope") + if not isinstance(execution_store, AuthoritySQLiteExecutionStore): + raise ValueError("authority and execution store are not colocated") + if ( + execution_store.scope_identity != scope_identity + or execution_store.owner_principal != authenticated_principal + or execution_store.authority_epoch != ledger["authority_epoch"] + ): + raise ValueError("host_capability_mismatch") + descriptor = self.profile_descriptor(execution_store, registry, configured) + descriptor.pop("installation_evidence") + return { + "profile_report_format": "determa.host_authority_profile_report", + "profile_report_schema_version": 1, + "scope_identity": scope_identity, + "authority_epoch": ledger["authority_epoch"], + "scope_generation": ledger["scope_generation"], + **descriptor, + } + + def allocate( + self, + scope_identity: str, + owner_principal: str, + *, + roots: tuple[str, ...] = (), + definition_references: tuple[str, ...] = (), + ) -> bool: + """Create trusted authority data and a permanent no-reuse marker.""" + if ( + type(scope_identity) is not str + or not scope_identity + or type(owner_principal) is not str + or not owner_principal + ): + raise ValueError("scope and owner identities are required") + if ( + any(type(item) is not str or not item for item in (*roots, *definition_references)) + or len(set(roots)) != len(roots) + or len(set(definition_references)) != len(definition_references) + ): + raise ValueError("invalid initial scope inventory") + ledger = { + "allocated_scope_identities": [scope_identity], + "scope_identity": scope_identity, + "owner_principal": owner_principal, + "authority_epoch": "0", + "scope_generation": "0", + "state": "active", + "receipts": [], + "mutation_bytes": [], + "checkpoint_bytes": [], + "journal_entries": [], + "ingress_acknowledgements": [], + "active_claims": [], + "roots": list(roots), + "tombstones": [], + "pending_intents": [], + "terminal_intents": [], + "definition_references": list(definition_references), + "migration_references": [], + "required_participant_records": [], + "freeze": None, + "inventory": [], + "retirement_grants": [], + "destination_activations": [], + } + return self._insert_ledger(ledger) + + def _insert_ledger(self, ledger: Mapping[str, Any]) -> bool: + """Seed trusted ledger storage; profile fixtures use this private seam.""" + scope = ledger["scope_identity"] + with self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + try: + connection.execute("INSERT INTO determa_scope_allocations VALUES (?)", (scope,)) + connection.execute( + "INSERT INTO determa_scope_authority VALUES (?, ?)", (scope, _compact(ledger)) + ) + except sqlite3.IntegrityError: + connection.rollback() + return False + connection.commit() + return True + + def inspect(self, scope_identity: str) -> dict[str, Any] | None: + with self._connect() as connection: + row = connection.execute( + "SELECT ledger FROM determa_scope_authority WHERE scope_identity = ?", + (scope_identity,), + ).fetchone() + return None if row is None else _parse(row[0]) + + def perform( + self, + request_bytes: str, + invocation: Mapping[str, Any], + *, + native_mutation_bytes: str | None = None, + ) -> str: + """Execute one host-authorized authority request and return committed bytes.""" + result = self._perform( + request_bytes, + invocation, + native_mutation_bytes=native_mutation_bytes, + ) + assert result is not None + return result + + def _perform( + self, + request_bytes: str, + invocation: Mapping[str, Any], + *, + native_mutation_bytes: str | None = None, + fault: Mapping[str, Any] | str | None = None, + before_commit: Callable[[], None] | None = None, + ) -> str | None: + """Execute one authenticated operation with a guard held through commit. + + ``native_mutation_bytes`` are proposed host bytes, never caller checkpoint + state. Hosted integration prepares them internally before calling this seam. + """ + try: + request = _parse(request_bytes) + code = _valid_request(request) + except (ValueError, TypeError, KeyError): + request, code = None, "invalid_host_request" + if code: + return _compact(_result(None, None, code, early=True)) + assert isinstance(request, dict) + scope = request["scope_identity"] + if ( + not invocation.get("authenticated_principal") + or scope not in invocation.get("authorized_scopes", ()) + or request["operation"] not in invocation.get("operation_rights", ()) + ): + return _compact(_result(request, None, "unauthorized_scope")) + try: + self.validate_schema() + except (OSError, sqlite3.Error, ValueError): + return _compact(_result(request, None, "host_capability_mismatch")) + with self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + "SELECT ledger FROM determa_scope_authority WHERE scope_identity = ?", (scope,) + ).fetchone() + if row is None: + return _compact(_result(request, None, "unauthorized_scope")) + ledger = _parse(row[0]) + operation = request["operation"] + if operation != "read_authority": + receipt = next( + ( + item + for item in ledger["receipts"] + if item["operation_id"] == request["operation_id"] + ), + None, + ) + if receipt is not None: + if ( + receipt["request_digest"] != request["request_digest"] + or receipt["request_bytes"] != request_bytes + ): + return _compact(_result(request, ledger, "scope_operation_conflict")) + if receipt["result_bytes"] is None: + return _compact(_result(request, ledger, "replay_evidence_expired")) + return str(receipt["result_bytes"]) + if operation == "read_authority": + return _compact(_result(request, ledger, None)) + if operation == "prove_retirement": + invalid_proof = any( + _parse(receipt["request_bytes"])["operation"] == "freeze_scope" + and receipt["request_digest"] == request["arguments"]["freeze_evidence_digest"] + for receipt in ledger["receipts"] + ) + return _compact( + _result( + request, + ledger, + "scope_fence_unproven" if invalid_proof else "host_capability_mismatch", + ) + ) + if operation == "fence_worker": + return _compact(_result(request, ledger, "host_capability_mismatch")) + if isinstance(fault, dict) and fault.get("epoch_check_separate_from_commit"): + return _compact(_result(request, ledger, "host_capability_mismatch")) + if ( + isinstance(fault, dict) + and fault.get("native_transaction_fate") == "unknown" + or fault == "commit_unknown_disconnect" + ): + ledger["state"] = "transaction_in_doubt" + connection.execute( + "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", + (_compact(ledger), scope), + ) + connection.commit() + return _compact(_result(request, ledger, "scope_transaction_in_doubt")) + if ledger["state"] == "transaction_in_doubt": + return _compact(_result(request, ledger, "scope_transaction_in_doubt")) + if request["expected_authority_epoch"] != ledger["authority_epoch"]: + return _compact(_result(request, ledger, "stale_scope_authority")) + if request["expected_scope_generation"] != ledger["scope_generation"]: + return _compact(_result(request, ledger, "scope_generation_conflict")) + if ( + ledger["state"] != "active" + or invocation["authenticated_principal"] != ledger["owner_principal"] + ): + return _compact(_result(request, ledger, "stale_scope_authority")) + if operation == "guarded_commit": + if ( + native_mutation_bytes is None + or "sha256:" + hashlib.sha256(native_mutation_bytes.encode()).hexdigest() + != request["arguments"]["mutation_digest"] + ): + return _compact(_result(request, ledger, "invalid_host_request")) + ledger["mutation_bytes"].append(native_mutation_bytes) + ledger["checkpoint_bytes"].append(native_mutation_bytes) + if operation == "freeze_scope": + if fault == "omit_receipt_from_inventory": + return _compact(_result(request, ledger, "scope_fence_unproven")) + if ( + ledger["required_participant_records"] + or ledger["journal_entries"] + or ledger["active_claims"] + ): + return _compact(_result(request, ledger, "host_capability_mismatch")) + ledger["scope_generation"] = str(int(ledger["scope_generation"]) + 1) + if operation == "freeze_scope": + ledger["state"] = "frozen" + ledger["active_claims"] = [] + response = _result(request, ledger, None) + ledger["receipts"].append( + { + "operation_id": request["operation_id"], + "request_digest": request["request_digest"], + "request_bytes": request_bytes, + "result_bytes": _compact(response), + "evidence_digest": response["evidence_digest"], + } + ) + if operation == "freeze_scope": + ledger["inventory"] = _inventory(ledger) + ledger["freeze"] = { + "evidence_digest": response["evidence_digest"], + "generation": ledger["scope_generation"], + "inventory_digest": hash_value( + [ + "determa-host-authority-frozen-inventory-1", + response["evidence_digest"], + ledger["inventory"], + ] + ), + "required_participants": copy.deepcopy(ledger["required_participant_records"]), + } + connection.execute( + "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", + (_compact(ledger), scope), + ) + if before_commit is not None: + before_commit() + if fault == "precommit_abort": + connection.rollback() + return None + connection.commit() + if fault == "drop_response_after_commit": + return None + return _compact(response) + + def _resolve_rolled_back_fate(self, scope_identity: str, operation_id: str) -> None: + """Clear doubt after the host has terminated the old session and checked storage.""" + with self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + "SELECT ledger FROM determa_scope_authority WHERE scope_identity = ?", + (scope_identity,), + ).fetchone() + if row is None: + raise ValueError("unknown scope") + ledger = _parse(row[0]) + if ledger["state"] != "transaction_in_doubt": + raise ValueError("scope has no uncertain transaction") + if any(item["operation_id"] == operation_id for item in ledger["receipts"]): + raise ValueError("operation committed; rollback not proved") + ledger["state"] = "active" + connection.execute( + "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", + (_compact(ledger), scope_identity), + ) + connection.commit() + + def _mark_transaction_in_doubt(self, scope_identity: str) -> None: + """Block freeze while a disconnected native writer's fate is unresolved.""" + with self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + "SELECT ledger FROM determa_scope_authority WHERE scope_identity = ?", + (scope_identity,), + ).fetchone() + if row is None: + raise ValueError("unknown scope") + ledger = _parse(row[0]) + if ledger["state"] != "active": + raise ValueError("scope is not active") + ledger["state"] = "transaction_in_doubt" + connection.execute( + "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", + (_compact(ledger), scope_identity), + ) + connection.commit() + + +class AuthoritySQLiteExecutionStore(SQLiteExecutionStore): + """Colocated execution store whose checkpoint writes hold the scope guard. + + Ordinary ``ExecutionHost`` operations use this adapter through the public + execution-store contract. The adapter owns the native SQLite transaction; no + portable host operation receives a raw connection or transaction. + """ + + def __init__( + self, + authority: SQLiteLocalAuthority, + scope_identity: str, + owner_principal: str, + authority_epoch: str, + *, + replay_retention: str = "bounded", + outbox_retention: str = "none", + ) -> None: + super().__init__( + authority.path, + journal_mode="WAL", + synchronous="FULL", + replay_retention=replay_retention, + outbox_retention=outbox_retention, + ) + self.authority = authority + self.scope_identity = scope_identity + self.owner_principal = owner_principal + self.authority_epoch = authority_epoch + + @contextmanager + def transaction(self, root_instance_id: str) -> Iterator[ExecutionStoreTransaction]: + """Commit checkpoint, authority generation and receipt atomically.""" + self.authority.validate_schema() + connection = self._connect() + try: + self._validate_schema(connection) + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + "SELECT ledger FROM determa_scope_authority WHERE scope_identity = ?", + (self.scope_identity,), + ).fetchone() + if row is None: + raise ExecutionStoreError("unauthorized_scope") + ledger = _parse(row[0]) + if ( + ledger["owner_principal"] != self.owner_principal + or ledger["authority_epoch"] != self.authority_epoch + or ledger["state"] != "active" + ): + raise ExecutionStoreError("stale_scope_authority") + if root_instance_id not in ledger["roots"]: + raise ExecutionStoreError("unauthorized_scope") + transaction = _SQLiteTransaction(connection, root_instance_id) + previous = transaction.load() + yield transaction + current = transaction.load() + if current != previous: + if current is None: + raise ExecutionStoreError("invalid_execution_checkpoint") + _, revision, digest = checkpoint_metadata(current) + operation_id = f"checkpoint:{root_instance_id}:{revision}:{digest}" + if any(item["operation_id"] == operation_id for item in ledger["receipts"]): + raise ExecutionStoreError("scope_operation_conflict") + mutation_digest = "sha256:" + hashlib.sha256(current).hexdigest() + request = { + "interface": _INTERFACE, + "interface_version": 1, + "operation": "guarded_commit", + "operation_id": operation_id, + "scope_identity": self.scope_identity, + "expected_authority_epoch": ledger["authority_epoch"], + "expected_scope_generation": ledger["scope_generation"], + "arguments": {"mutation_digest": mutation_digest}, + } + request["request_digest"] = hash_value( + ["determa-host-authority-request-1", request] + ) + ledger["scope_generation"] = str(int(ledger["scope_generation"]) + 1) + response = _result(request, ledger, None) + ledger["receipts"].append( + { + "operation_id": operation_id, + "request_digest": request["request_digest"], + "request_bytes": _compact(request), + "result_bytes": _compact(response), + "evidence_digest": response["evidence_digest"], + } + ) + encoded = current.decode("utf-8", "strict") + ledger["mutation_bytes"].append(encoded) + ledger["checkpoint_bytes"].append(encoded) + connection.execute( + "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", + (_compact(ledger), self.scope_identity), + ) + connection.commit() + except BaseException: + connection.rollback() + raise + finally: + connection.close() + + +def _authority_closure() -> bytes: + source_directory = Path(__file__).parent + return b"".join( + path.relative_to(source_directory).as_posix().encode() + b"\0" + path.read_bytes() + for path in sorted(source_directory.rglob("*")) + if path.is_file() and path.suffix in {".py", ".json"} + ) + + +def bundled_authority_descriptor() -> dict[str, Any]: + """Exact public descriptor for the installed local SQLite provider.""" + return { + "category": "authority", + "provider_reference": { + "identifier": "reference.sqlite-authority", + "version": __version__, + "content_digest": "sha256:" + hashlib.sha256(_authority_closure()).hexdigest(), + }, + "interface_version": 1, + "supported_capabilities": [ + "authoritative_scope_fencing", + "consistent_scope_inventory", + ], + } + + +class _BundledAuthorityProvider: + def validate_configuration(self, configuration: Mapping[str, Any]) -> dict[str, Any]: + if set(configuration) != { + "instance_id", + "path", + "scope_identity", + "owner_principal", + "authority_epoch", + "replay_retention", + "outbox_retention", + } or any( + type(configuration[key]) is not str or not configuration[key] for key in configuration + ): + raise ValueError("invalid_extension_configuration") + if configuration["instance_id"] != "local-authority": + raise ValueError("invalid_extension_configuration") + authority = SQLiteLocalAuthority(configuration["path"]) + if authority.path != configuration["path"]: + raise ValueError("invalid_extension_configuration") + store = AuthoritySQLiteExecutionStore( + authority, + configuration["scope_identity"], + configuration["owner_principal"], + configuration["authority_epoch"], + replay_retention=configuration["replay_retention"], + outbox_retention=configuration["outbox_retention"], + ) + return {"instance_id": configuration["instance_id"], "authority": authority, "store": store} + + def capabilities(self, instance: Any) -> list[str]: + return ["authoritative_scope_fencing", "consistent_scope_inventory"] + + def health(self, instance: Any) -> str: + authority = instance["authority"] + store = instance["store"] + try: + authority.validate_schema() + store.validate_schema() + ledger = authority.inspect(store.scope_identity) + if ( + type(authority) is not SQLiteLocalAuthority + or type(store) is not AuthoritySQLiteExecutionStore + or store.authority is not authority + or store.path != authority.path + or ledger is None + or ledger["owner_principal"] != store.owner_principal + or ledger["authority_epoch"] != store.authority_epoch + or ledger["state"] not in ("active", "frozen") + ): + return "unavailable" + except (ValueError, sqlite3.Error, KeyError): + return "unavailable" + return "healthy" + + +def bundled_sqlite_authority_provider_factory() -> _BundledAuthorityProvider: + return _BundledAuthorityProvider() + + +def configure_bundled_sqlite_authority( + registry: ExtensionRegistry, + path: str | Path, + scope_identity: str, + owner_principal: str, + authority_epoch: str, + *, + replay_retention: str = "bounded", + outbox_retention: str = "none", +) -> tuple[ConfiguredExtension, SQLiteLocalAuthority, AuthoritySQLiteExecutionStore]: + """Configure and return the exact authority/store instance used for host operations.""" + configuration = { + "instance_id": "local-authority", + "path": str(Path(path).resolve()), + "scope_identity": scope_identity, + "owner_principal": owner_principal, + "authority_epoch": authority_epoch, + "replay_retention": replay_retention, + "outbox_retention": outbox_retention, + } + configured = registry.validate_configuration(bundled_authority_descriptor(), configuration) + _, _, instance, _ = registry._bound(configured) + return configured, instance["authority"], instance["store"] diff --git a/src/determa/state/data/host-authority-operation-v1.schema.json b/src/determa/state/data/host-authority-operation-v1.schema.json new file mode 100644 index 0000000..00465ed --- /dev/null +++ b/src/determa/state/data/host-authority-operation-v1.schema.json @@ -0,0 +1,610 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://determa.dev/state/schema/host-authority-operation-v1.schema.json", + "title": "Optional host scope authority operation and result", + "oneOf": [ + { + "$ref": "#/$defs/request" + }, + { + "$ref": "#/$defs/result" + }, + { + "$ref": "#/$defs/earlyError" + } + ], + "$defs": { + "identity": { + "type": "string", + "minLength": 1 + }, + "decimal": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "operation": { + "enum": [ + "read_authority", + "guarded_commit", + "freeze_scope", + "fence_worker", + "prove_retirement" + ] + }, + "emptyArguments": { + "type": "object", + "properties": {}, + "additionalProperties": false + }, + "guardedCommitArguments": { + "type": "object", + "required": [ + "mutation_digest" + ], + "properties": { + "mutation_digest": { + "$ref": "#/$defs/digest" + } + }, + "additionalProperties": false + }, + "fenceWorkerArguments": { + "type": "object", + "required": [ + "root_instance_id", + "work_kind", + "work_identity", + "operation_token", + "expected_attempt_fence", + "expected_worker_principal" + ], + "properties": { + "root_instance_id": { + "$ref": "#/$defs/identity" + }, + "work_kind": { + "const": "effect" + }, + "work_identity": { + "$ref": "#/$defs/identity" + }, + "operation_token": { + "$ref": "#/$defs/identity" + }, + "expected_attempt_fence": { + "oneOf": [ + { + "$ref": "#/$defs/decimal" + }, + { + "type": "null" + } + ] + }, + "expected_worker_principal": { + "$ref": "#/$defs/identity" + } + }, + "additionalProperties": false + }, + "proveRetirementArguments": { + "type": "object", + "required": [ + "destination_binding_digest", + "freeze_evidence_digest" + ], + "properties": { + "destination_binding_digest": { + "$ref": "#/$defs/digest" + }, + "freeze_evidence_digest": { + "$ref": "#/$defs/digest" + } + }, + "additionalProperties": false + }, + "request": { + "type": "object", + "required": [ + "interface", + "interface_version", + "operation", + "operation_id", + "scope_identity", + "expected_authority_epoch", + "expected_scope_generation", + "request_digest", + "arguments" + ], + "properties": { + "interface": { + "const": "determa.host_authority" + }, + "interface_version": { + "const": 1 + }, + "operation": { + "$ref": "#/$defs/operation" + }, + "operation_id": { + "$ref": "#/$defs/identity" + }, + "scope_identity": { + "$ref": "#/$defs/identity" + }, + "expected_authority_epoch": { + "oneOf": [ + { + "$ref": "#/$defs/decimal" + }, + { + "type": "null" + } + ] + }, + "expected_scope_generation": { + "oneOf": [ + { + "$ref": "#/$defs/decimal" + }, + { + "type": "null" + } + ] + }, + "request_digest": { + "$ref": "#/$defs/digest" + }, + "arguments": { + "type": "object" + } + }, + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "operation": { + "const": "read_authority" + } + } + }, + "then": { + "properties": { + "arguments": { + "$ref": "#/$defs/emptyArguments" + }, + "expected_authority_epoch": { + "type": "null" + }, + "expected_scope_generation": { + "type": "null" + } + } + }, + "else": { + "properties": { + "expected_authority_epoch": { + "$ref": "#/$defs/decimal" + }, + "expected_scope_generation": { + "$ref": "#/$defs/decimal" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "guarded_commit" + } + } + }, + "then": { + "properties": { + "arguments": { + "$ref": "#/$defs/guardedCommitArguments" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "freeze_scope" + } + } + }, + "then": { + "properties": { + "arguments": { + "$ref": "#/$defs/emptyArguments" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "fence_worker" + } + } + }, + "then": { + "properties": { + "arguments": { + "$ref": "#/$defs/fenceWorkerArguments" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "prove_retirement" + } + } + }, + "then": { + "properties": { + "arguments": { + "$ref": "#/$defs/proveRetirementArguments" + } + } + } + } + ] + }, + "result": { + "type": "object", + "required": [ + "interface", + "interface_version", + "operation", + "operation_id", + "status", + "scope_identity", + "authority_epoch", + "scope_generation", + "state", + "evidence_digest", + "error_code", + "claim" + ], + "properties": { + "interface": { + "const": "determa.host_authority" + }, + "interface_version": { + "const": 1 + }, + "operation": { + "$ref": "#/$defs/operation" + }, + "operation_id": { + "$ref": "#/$defs/identity" + }, + "status": { + "enum": [ + "accepted", + "rejected" + ] + }, + "scope_identity": { + "oneOf": [ + { + "$ref": "#/$defs/identity" + }, + { + "type": "null" + } + ] + }, + "authority_epoch": { + "oneOf": [ + { + "$ref": "#/$defs/decimal" + }, + { + "type": "null" + } + ] + }, + "scope_generation": { + "oneOf": [ + { + "$ref": "#/$defs/decimal" + }, + { + "type": "null" + } + ] + }, + "state": { + "enum": [ + "active", + "freezing", + "frozen", + "retired", + "transaction_in_doubt", + null + ] + }, + "evidence_digest": { + "oneOf": [ + { + "$ref": "#/$defs/digest" + }, + { + "type": "null" + } + ] + }, + "error_code": { + "enum": [ + null, + "host_capability_mismatch", + "unauthorized_scope", + "scope_not_active", + "stale_scope_authority", + "scope_operation_conflict", + "scope_generation_conflict", + "scope_transaction_in_doubt", + "scope_fence_unproven", + "replay_evidence_expired", + "stale_attempt_fence", + "worker_principal_mismatch" + ] + }, + "claim": { + "oneOf": [ + { + "$ref": "#/$defs/workerClaim" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "accepted" + } + } + }, + "then": { + "properties": { + "evidence_digest": { + "$ref": "#/$defs/digest" + }, + "error_code": { + "type": "null" + } + } + } + }, + { + "if": { + "properties": { + "status": { + "const": "rejected" + } + } + }, + "then": { + "properties": { + "evidence_digest": { + "type": "null" + }, + "error_code": { + "not": { + "type": "null" + } + } + } + } + }, + { + "if": { + "properties": { + "error_code": { + "const": "unauthorized_scope" + } + } + }, + "then": { + "properties": { + "scope_identity": { + "type": "null" + }, + "authority_epoch": { + "type": "null" + }, + "scope_generation": { + "type": "null" + }, + "state": { + "type": "null" + } + } + }, + "else": { + "properties": { + "scope_identity": { + "$ref": "#/$defs/identity" + }, + "authority_epoch": { + "$ref": "#/$defs/decimal" + }, + "scope_generation": { + "$ref": "#/$defs/decimal" + }, + "state": { + "enum": [ + "active", + "freezing", + "frozen", + "retired", + "transaction_in_doubt" + ] + } + } + } + }, + { + "if": { + "properties": { + "status": { + "const": "accepted" + }, + "operation": { + "const": "fence_worker" + } + }, + "required": [ + "status", + "operation" + ] + }, + "then": { + "properties": { + "claim": { + "$ref": "#/$defs/workerClaim" + } + } + }, + "else": { + "properties": { + "claim": { + "type": "null" + } + } + } + } + ] + }, + "workerClaim": { + "type": "object", + "required": [ + "scope_identity", + "root_instance_id", + "work_kind", + "work_identity", + "operation_token", + "scope_authority_epoch", + "attempt_fence", + "worker_principal", + "expires_at", + "state" + ], + "properties": { + "scope_identity": { + "$ref": "#/$defs/identity" + }, + "root_instance_id": { + "$ref": "#/$defs/identity" + }, + "work_kind": { + "const": "effect" + }, + "work_identity": { + "$ref": "#/$defs/identity" + }, + "operation_token": { + "$ref": "#/$defs/identity" + }, + "scope_authority_epoch": { + "$ref": "#/$defs/decimal" + }, + "attempt_fence": { + "$ref": "#/$defs/decimal" + }, + "worker_principal": { + "$ref": "#/$defs/identity" + }, + "expires_at": { + "type": "string", + "pattern": "^(0|-?[1-9][0-9]*)$", + "description": "Canonical signed Unix-epoch nanoseconds; normative signed 64-bit bounds are checked semantically." + }, + "state": { + "const": "active" + } + }, + "additionalProperties": false + }, + "earlyError": { + "type": "object", + "required": [ + "interface", + "interface_version", + "operation", + "operation_id", + "status", + "scope_identity", + "authority_epoch", + "scope_generation", + "state", + "evidence_digest", + "error_code", + "claim" + ], + "properties": { + "interface": { + "const": "determa.host_authority" + }, + "interface_version": { + "const": 1 + }, + "operation": { + "type": "null" + }, + "operation_id": { + "type": "null" + }, + "status": { + "const": "rejected" + }, + "scope_identity": { + "type": "null" + }, + "authority_epoch": { + "type": "null" + }, + "scope_generation": { + "type": "null" + }, + "state": { + "type": "null" + }, + "evidence_digest": { + "type": "null" + }, + "error_code": { + "enum": [ + "invalid_host_request", + "unsupported_host_protocol", + "unsupported_host_protocol_version" + ] + }, + "claim": { + "type": "null" + } + }, + "additionalProperties": false + } + } +} diff --git a/src/determa/state/data/host-authority-profile-report-v1.schema.json b/src/determa/state/data/host-authority-profile-report-v1.schema.json new file mode 100644 index 0000000..c02b027 --- /dev/null +++ b/src/determa/state/data/host-authority-profile-report-v1.schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://determa.dev/state/schema/host-authority-profile-report-v1.schema.json", + "title": "Composed host authority profile report version 1", + "type": "object", + "required": ["profile_report_format", "profile_report_schema_version", "scope_identity", "authority_epoch", "scope_generation", "extension_report", "authority_storage_boundary", "topology", "source_binding_digest", "destination_binding_digest", "required_participants", "guarantees"], + "properties": { + "profile_report_format": { "const": "determa.host_authority_profile_report" }, + "profile_report_schema_version": { "const": 1 }, + "scope_identity": { "type": "string", "minLength": 1 }, + "authority_epoch": { "oneOf": [{ "$ref": "#/$defs/decimal" }, { "type": "null" }] }, + "scope_generation": { "oneOf": [{ "$ref": "#/$defs/decimal" }, { "type": "null" }] }, + "extension_report": { + "oneOf": [ + { "allOf": [ + { "$ref": "extension-capability-report-v1.schema.json" }, + { "properties": { "category": { "const": "authority" } }, "required": ["category"] } + ] }, + { "type": "null" } + ] + }, + "authority_storage_boundary": { "oneOf": [{ "type": "string", "minLength": 1 }, { "type": "null" }] }, + "topology": { + "type": "object", + "required": ["identifier", "configuration_digest", "process_boundary", "host_boundary"], + "properties": { + "identifier": { "type": "string", "minLength": 1 }, + "configuration_digest": { "$ref": "#/$defs/digest" }, + "process_boundary": { "type": "string", "minLength": 1 }, + "host_boundary": { "type": "string", "minLength": 1 } + }, + "additionalProperties": false + }, + "source_binding_digest": { "$ref": "#/$defs/digest" }, + "destination_binding_digest": { "oneOf": [{ "$ref": "#/$defs/digest" }, { "type": "null" }] }, + "required_participants": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "required": ["role", "provider_reference", "instance_id"], + "properties": { + "role": { "enum": ["journal", "worker", "archive", "helper"] }, + "provider_reference": { "$ref": "provider-reference-v1.schema.json" }, + "instance_id": { "type": "string", "minLength": 1 } + }, + "additionalProperties": false + } + }, + "guarantees": { + "type": "object", + "required": ["guarded_local_writes", "worker_fencing", "complete_scope_inventory", "safe_relocation"], + "properties": { + "guarded_local_writes": { "type": "boolean" }, + "worker_fencing": { "type": "boolean" }, + "complete_scope_inventory": { "type": "boolean" }, + "safe_relocation": { "type": "boolean" } + }, + "additionalProperties": false + } + }, + "additionalProperties": false, + "allOf": [ + { + "if": { "properties": { "extension_report": { "type": "null" } } }, + "then": { "properties": { "authority_epoch": { "type": "null" }, "scope_generation": { "type": "null" }, "authority_storage_boundary": { "type": "null" }, "guarantees": { "properties": { "guarded_local_writes": { "const": false }, "worker_fencing": { "const": false }, "complete_scope_inventory": { "const": false }, "safe_relocation": { "const": false } } } } }, + "else": { "properties": { "authority_epoch": { "$ref": "#/$defs/decimal" }, "scope_generation": { "$ref": "#/$defs/decimal" }, "authority_storage_boundary": { "type": "string", "minLength": 1 } } } + }, + { + "if": { "properties": { "guarantees": { "properties": { "guarded_local_writes": { "const": true } }, "required": ["guarded_local_writes"] } } }, + "then": { "properties": { "extension_report": { "type": "object", "properties": { "health": { "const": "healthy" }, "claims": { "contains": { "const": "authoritative_scope_fencing" } } } } } } + }, + { + "if": { "properties": { "guarantees": { "properties": { "worker_fencing": { "const": true } }, "required": ["worker_fencing"] } } }, + "then": { "properties": { "guarantees": { "properties": { "guarded_local_writes": { "const": true } } } } } + }, + { + "if": { "properties": { "guarantees": { "properties": { "complete_scope_inventory": { "const": true } }, "required": ["complete_scope_inventory"] } } }, + "then": { "properties": { "extension_report": { "type": "object", "properties": { "health": { "const": "healthy" }, "claims": { "contains": { "const": "consistent_scope_inventory" } } } } } } + }, + { + "if": { "properties": { "guarantees": { "properties": { "safe_relocation": { "const": true } }, "required": ["safe_relocation"] } } }, + "then": { + "properties": { + "destination_binding_digest": { "$ref": "#/$defs/digest" }, + "guarantees": { "properties": { "guarded_local_writes": { "const": true }, "complete_scope_inventory": { "const": true } } }, + "extension_report": { "type": "object", "properties": { "health": { "const": "healthy" }, "claims": { "contains": { "const": "safe_relocation" } } } } + } + } + } + ], + "$defs": { + "decimal": { "type": "string", "pattern": "^(0|[1-9][0-9]*)$" }, + "digest": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" } + } +} diff --git a/src/determa/state/extensions.py b/src/determa/state/extensions.py index 8022364..fc2aa4b 100644 --- a/src/determa/state/extensions.py +++ b/src/determa/state/extensions.py @@ -233,7 +233,7 @@ def validate_configuration( configured = ConfiguredExtension(self._token, registered, clean, provider, instance) store = ( instance.get("store") - if isinstance(provider, _BundledStoreProvider) and isinstance(instance, Mapping) + if isinstance(instance, Mapping) and _is_bundled_provider(provider) else None ) self._bindings[id(configured)] = _InstanceBinding( @@ -278,7 +278,7 @@ def _bound( ) if instance_id != configured._configuration.get("instance_id"): raise ExtensionError(Code.INVALID_EXTENSION_CONFIGURATION) - if isinstance(configured._provider, _BundledStoreProvider): + if _is_bundled_provider(configured._provider): store = instance.get("store") if isinstance(instance, Mapping) else None if ( store is not binding.store @@ -287,6 +287,28 @@ def _bound( or _shadows_executable(store) ): raise ExtensionError(Code.EXTENSION_IDENTITY_MISMATCH) + from .authority import ( + AuthoritySQLiteExecutionStore, + SQLiteLocalAuthority, + _BundledAuthorityProvider, + ) + + if isinstance(configured._provider, _BundledAuthorityProvider): + authority = instance.get("authority") if isinstance(instance, Mapping) else None + if ( + not isinstance(authority, SQLiteLocalAuthority) + or not isinstance(store, AuthoritySQLiteExecutionStore) + or type(authority) is not SQLiteLocalAuthority + or type(store) is not AuthoritySQLiteExecutionStore + or _shadows_executable(authority) + or authority.path != configured._configuration.get("path") + or store.scope_identity != configured._configuration.get("scope_identity") + or store.owner_principal != configured._configuration.get("owner_principal") + or store.authority_epoch != configured._configuration.get("authority_epoch") + or store.replay_retention != configured._configuration.get("replay_retention") + or store.outbox_retention != configured._configuration.get("outbox_retention") + ): + raise ExtensionError(Code.EXTENSION_IDENTITY_MISMATCH) return descriptor, configured._provider, instance, evaluator def capabilities(self, configured: ConfiguredExtension, *extra: Any) -> list[str]: @@ -318,6 +340,14 @@ def negotiate( """Evaluate current configured instance before any core or host mutation.""" del lookup_uri # URI is only a resolver hint; it cannot change exact identity. configured = self.validate_configuration(descriptor, configuration) + return self.report(configured, requirement) + + def report( + self, + configured: ConfiguredExtension, + requirement: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + """Recheck source, identity, current health and proof for a configured instance.""" claims = self.capabilities(configured) health = self.health(configured) checked, _, instance, evaluator = self._bound(configured) @@ -486,6 +516,12 @@ def health(self, instance: Any) -> str: return "healthy" if instance["store"].health().get("healthy") is True else "unavailable" +def _is_bundled_provider(provider: Any) -> bool: + from .authority import _BundledAuthorityProvider + + return isinstance(provider, (_BundledStoreProvider, _BundledAuthorityProvider)) + + def _shadows_executable(instance: Any) -> bool: own = vars(instance) return any( @@ -522,13 +558,21 @@ def _bundled_factory_matches_source(name: str, factory: Any) -> bool: import types package = Path(__file__).parent - module = importlib.import_module(f"determa.state.stores.{name}") + authority = name == "authority" + module = importlib.import_module( + "determa.state.authority" if authority else f"determa.state.stores.{name}" + ) if not isinstance(factory, types.FunctionType): return False - if factory is not getattr( - module, f"{name}_execution_store_factory", None - ) or factory is not getattr( - importlib.import_module("determa.state.stores"), f"{name}_execution_store_factory", None + factory_name = ( + "bundled_sqlite_authority_provider_factory" + if authority + else f"{name}_execution_store_factory" + ) + if factory is not getattr(module, factory_name, None) or ( + not authority + and factory + is not getattr(importlib.import_module("determa.state.stores"), factory_name, None) ): return False @@ -654,11 +698,11 @@ def verify_function(candidate: Any, code: types.CodeType) -> bool: return True return verify_function(value, expected) - for origin in ( - importlib.import_module(__name__), - module, - importlib.import_module("determa.state.stores.base"), - ): + origins = [importlib.import_module(__name__), module] + if authority: + origins.extend((importlib.import_module("determa.state.stores.sqlite"),)) + origins.append(importlib.import_module("determa.state.stores.base")) + for origin in origins: top = source_code(origin) if top is None: return False @@ -666,6 +710,30 @@ def verify_function(candidate: Any, code: types.CodeType) -> bool: if isinstance(child, types.CodeType) and not child.co_name.startswith("<"): if not verify_definition(origin, child.co_name): return False + if authority: + host = importlib.import_module("determa.state.host") + host_code = source_code(host) + if host_code is None: + return False + for child in host_code.co_consts: + if not isinstance(child, types.CodeType) or child.co_name.startswith("<"): + continue + loaded = vars(host).get(child.co_name) + if isinstance(loaded, type): + for method_code in child.co_consts: + if not isinstance( + method_code, types.CodeType + ) or method_code.co_name.startswith("<"): + continue + method = vars(loaded).get(method_code.co_name) + if isinstance(method, (staticmethod, classmethod)): + method = method.__func__ + if isinstance(method, property): + method = method.fget + if getattr(method, "__code__", None) != method_code: + return False + elif getattr(loaded, "__code__", None) != child: + return False return True @@ -716,6 +784,14 @@ def bundled_extension_registry( installed by a host. Memory's `ephemeral` only states that loss is permitted. """ from .__about__ import __version__ + from .authority import ( + AuthoritySQLiteExecutionStore, + SQLiteLocalAuthority, + _authority_closure, + _BundledAuthorityProvider, + bundled_authority_descriptor, + bundled_sqlite_authority_provider_factory, + ) from .stores import ( file_execution_store_factory, memory_execution_store_factory, @@ -728,6 +804,27 @@ def bundled_extension_registry( def verify(provider: Any, descriptor: Mapping[str, Any]) -> bool: reference = descriptor["provider_reference"] identifier = reference["identifier"] + if identifier == "reference.sqlite-authority": + import hashlib + + installed_authority = ( + descriptor == bundled_authority_descriptor() + and reference["content_digest"] + == "sha256:" + hashlib.sha256(_authority_closure()).hexdigest() + ) + if isinstance(provider, _BundledAuthorityProvider): + return ( + installed_authority + and _bundled_factory_matches_source( + "authority", bundled_sqlite_authority_provider_factory + ) + and not _shadows_executable(provider) + ) + return ( + installed_authority + and provider is bundled_sqlite_authority_provider_factory + and _bundled_factory_matches_source("authority", provider) + ) installed = ( identifier in { @@ -812,4 +909,36 @@ def make_provider( registry.register(descriptor, make_provider) if evaluator is not None: registry._set_operational_evaluator(descriptor, evaluator) + authority_descriptor = bundled_authority_descriptor() + registry.register(authority_descriptor, bundled_sqlite_authority_provider_factory) + + def authority_proof(instance: Any, health: str, claims: Sequence[str]) -> frozenset[str]: + authority = instance.get("authority") if isinstance(instance, dict) else None + store = instance.get("store") if isinstance(instance, dict) else None + if ( + health != "healthy" + or type(authority) is not SQLiteLocalAuthority + or type(store) is not AuthoritySQLiteExecutionStore + or store.authority is not authority + or store.path != authority.path + or store.journal_mode != "WAL" + or store.synchronous != "FULL" + ): + return frozenset() + try: + authority.validate_schema() + store.validate_schema() + ledger = authority.inspect(store.scope_identity) + except (ValueError, KeyError): + return frozenset() + if ( + ledger is None + or ledger["owner_principal"] != store.owner_principal + or ledger["authority_epoch"] != store.authority_epoch + or ledger["state"] not in ("active", "frozen") + ): + return frozenset() + return frozenset(claims) & frozenset(authority_descriptor["supported_capabilities"]) + + registry._set_operational_evaluator(authority_descriptor, authority_proof) return registry diff --git a/tests/test_local_authority.py b/tests/test_local_authority.py new file mode 100644 index 0000000..a4ecdca --- /dev/null +++ b/tests/test_local_authority.py @@ -0,0 +1,246 @@ +"""Durable safety checks for the optional single-database authority.""" + +from __future__ import annotations + +import hashlib +import json +import threading +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from determa.state import ( + AuthoritySQLiteExecutionStore, + ExecutionHost, + ExecutionHostError, + MemoryArtifactResolver, + SQLiteExecutionStore, + SQLiteLocalAuthority, + load_bundle, +) +from determa.state.authority import configure_bundled_sqlite_authority +from determa.state.extensions import ExtensionError, bundled_extension_registry +from determa.state.stores.base import ExecutionStoreError +from determa.state.wire import hash_value + + +def _request(operation: str, operation_id: str, generation: str, arguments: dict) -> str: + request = { + "interface": "determa.host_authority", + "interface_version": 1, + "operation": operation, + "operation_id": operation_id, + "scope_identity": "scope-1", + "expected_authority_epoch": "0", + "expected_scope_generation": generation, + "arguments": arguments, + } + request["request_digest"] = hash_value(["determa-host-authority-request-1", request]) + return json.dumps(request, sort_keys=True, separators=(",", ":")) + + +def test_lost_response_replays_committed_receipt_and_freeze_fences_writer(tmp_path) -> None: + path = tmp_path / "authority.sqlite" + authority = SQLiteLocalAuthority(path) + authority.setup_schema() + assert authority.allocate( + "scope-1", "owner-1", roots=("root-1",), definition_references=("definition-1",) + ) + registry = bundled_extension_registry(include_postgresql=False) + configured, authority, store = configure_bundled_sqlite_authority( + registry, path, "scope-1", "owner-1", "0" + ) + store.setup_schema() + report = authority.profile_report("scope-1", "owner-1", store, registry, configured) + assert report["guarantees"] == { + "guarded_local_writes": True, + "worker_fencing": False, + "complete_scope_inventory": True, + "safe_relocation": False, + } + with pytest.raises(ValueError, match="unauthorized_scope"): + authority.profile_report("scope-1", "another-principal", store, registry, configured) + with pytest.raises(ValueError, match="not colocated"): + authority.profile_report( + "scope-1", "owner-1", SQLiteExecutionStore(authority.path), registry, configured + ) + original_transaction = AuthoritySQLiteExecutionStore.transaction + try: + AuthoritySQLiteExecutionStore.transaction = lambda self, root: None # type: ignore[method-assign] + with pytest.raises(ExtensionError, match="extension_identity_mismatch"): + registry.report(configured) + finally: + AuthoritySQLiteExecutionStore.transaction = original_transaction + original_create = ExecutionHost.create_v1 + try: + ExecutionHost.create_v1 = lambda self, *args: None # type: ignore[method-assign] + with pytest.raises(ExtensionError, match="extension_identity_mismatch"): + registry.report(configured) + finally: + ExecutionHost.create_v1 = original_create + invocation = { + "authenticated_principal": "owner-1", + "authorized_scopes": ["scope-1"], + "operation_rights": ["guarded_commit", "freeze_scope"], + } + mutation = '{"checkpoint":"one"}' + commit = _request( + "guarded_commit", + "commit-1", + "0", + {"mutation_digest": "sha256:" + hashlib.sha256(mutation.encode()).hexdigest()}, + ) + assert ( + authority._perform( + commit, invocation, native_mutation_bytes=mutation, fault="drop_response_after_commit" + ) + is None + ) + restarted = SQLiteLocalAuthority(path) + replay = restarted.perform(commit, invocation, native_mutation_bytes=mutation) + assert replay is not None and json.loads(replay)["status"] == "accepted" + assert restarted.inspect("scope-1")["checkpoint_bytes"] == [mutation] + freeze = _request("freeze_scope", "freeze-1", "1", {}) + assert json.loads(restarted.perform(freeze, invocation))["state"] == "frozen" + second = _request( + "guarded_commit", + "commit-2", + "2", + {"mutation_digest": "sha256:" + hashlib.sha256(mutation.encode()).hexdigest()}, + ) + assert ( + json.loads(restarted.perform(second, invocation, native_mutation_bytes=mutation))["status"] + == "rejected" + ) + + +def test_allocation_marker_survives_checkpoint_removal(tmp_path) -> None: + authority = SQLiteLocalAuthority(tmp_path / "authority.sqlite") + authority.setup_schema() + assert authority.allocate("scope-1", "owner-1") + assert not authority.allocate("scope-1", "owner-1") + original = authority.inspect("scope-1") + assert original is not None + original["checkpoint_bytes"] = ["checkpoint"] + with authority._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + connection.execute( + "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", + (json.dumps({**original, "checkpoint_bytes": []}), "scope-1"), + ) + connection.commit() + assert not SQLiteLocalAuthority(authority.path).allocate("scope-1", "owner-1") + + +def test_public_checkpoint_host_commits_under_same_scope_guard(tmp_path) -> None: + machine = """ +format: 1 +namespace: test.authority +machines: + - machine_id: simple + version: 1 + root: { type: simple } +""" + bundle = load_bundle(machine) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + authority = SQLiteLocalAuthority(tmp_path / "colocated.sqlite") + authority.setup_schema() + assert authority.allocate( + "scope-1", + "owner-1", + roots=("root-1",), + definition_references=(bundle.fingerprint,), + ) + registry = bundled_extension_registry(include_postgresql=False) + configured, authority, store = configure_bundled_sqlite_authority( + registry, authority.path, "scope-1", "owner-1", "0" + ) + store.setup_schema() + assert set(registry.report(configured)["claims"]) == { + "authoritative_scope_fencing", + "consistent_scope_inventory", + } + host = ExecutionHost(store, resolver) + host.create_v1(bundle, "simple", "root-1", "create-1", {}) + checkpoint = host.read_checkpoint("root-1") + assert checkpoint is not None + ledger = authority.inspect("scope-1") + assert ledger is not None + assert ledger["scope_generation"] == "1" + assert ledger["receipts"][0]["result_bytes"] is not None + assert ledger["checkpoint_bytes"] == [checkpoint.canonical_bytes.decode()] + with pytest.raises(ExecutionHostError, match="creation_id_conflict"): + host.create_v1(bundle, "simple", "root-1", "create-2", {}) + assert authority.inspect("scope-1")["scope_generation"] == "1" + with store.transaction("root-1") as transaction: + assert transaction.load() == checkpoint.canonical_bytes + assert authority.inspect("scope-1")["scope_generation"] == "1" + + invocation = { + "authenticated_principal": "owner-1", + "authorized_scopes": ["scope-1"], + "operation_rights": ["freeze_scope"], + } + response = authority.perform(_request("freeze_scope", "freeze-1", "1", {}), invocation) + assert set(registry.report(configured)["claims"]) == { + "authoritative_scope_fencing", + "consistent_scope_inventory", + } + assert authority.profile_report("scope-1", "owner-1", store, registry, configured)[ + "guarantees" + ]["complete_scope_inventory"] + assert json.loads(response)["state"] == "frozen" + with pytest.raises(ExecutionStoreError, match="stale_scope_authority"): + host.read_checkpoint("root-1") + with store._connect() as connection: + actual = connection.execute( + "SELECT checkpoint FROM determa_execution_checkpoints WHERE root_instance_id = ?", + ("root-1",), + ).fetchone() + assert actual is not None and bytes(actual[0]) == checkpoint.canonical_bytes + + +def test_public_checkpoint_commit_serializes_with_freeze(tmp_path) -> None: + bundle = load_bundle( + """format: 1 +namespace: test.guard_race +machines: + - machine_id: simple + version: 1 + root: { type: simple } +""" + ) + authority = SQLiteLocalAuthority(tmp_path / "race.sqlite") + authority.setup_schema() + assert authority.allocate("scope-1", "owner-1", roots=("root-1",)) + store = AuthoritySQLiteExecutionStore(authority, "scope-1", "owner-1", "0") + store.setup_schema() + entered = threading.Event() + release = threading.Event() + + def pause_before_native_commit(boundary: str) -> None: + if boundary == "before_commit": + entered.set() + assert release.wait(5) + + host = ExecutionHost( + store, + MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}), + fault_injector=pause_before_native_commit, + ) + invocation = { + "authenticated_principal": "owner-1", + "authorized_scopes": ["scope-1"], + "operation_rights": ["freeze_scope"], + } + with ThreadPoolExecutor(max_workers=2) as pool: + writer = pool.submit(host.create_v1, bundle, "simple", "root-1", "create-1", {}) + assert entered.wait(5) + freezer = pool.submit( + authority.perform, _request("freeze_scope", "freeze-1", "1", {}), invocation + ) + assert not freezer.done() + release.set() + assert writer.result(timeout=5)["result"] == "committed" + assert json.loads(freezer.result(timeout=5))["state"] == "frozen" + assert authority.inspect("scope-1")["scope_generation"] == "2" From 8f9cf4ecdc6161a8bfd0527eb2059ee1f7b8a59b Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 05:17:01 +0000 Subject: [PATCH 02/30] Recover retained worker authority patches before effect reconstruction Recover complete original authority, worker adapter and package edits from retained coding-agent patches. Copy the six effect schemas from approved specification86bb88dd. Unfinished recovery checkpoint: effect implementation and adapter additions are truncated in the retained record and are not yet reconstructed. Integration with reviewed main and all validation/reviews remain; draft only. --- conformance/authority_adapter.py | 64 ++- conformance/worker_authority_adapter.py | 17 + pyproject.toml | 6 + src/determa/state/authority.py | 246 +++++++++-- ...effect-cancellation-request-v1.schema.json | 30 ++ ...ffect-cancellation-response-v1.schema.json | 186 +++++++++ .../data/effect-result-request-v1.schema.json | 46 ++ .../effect-result-response-v1.schema.json | 173 ++++++++ .../data/host-effect-claim-v1.schema.json | 69 +++ .../data/host-effect-journal-v1.schema.json | 393 ++++++++++++++++++ 10 files changed, 1195 insertions(+), 35 deletions(-) create mode 100644 conformance/worker_authority_adapter.py create mode 100644 src/determa/state/data/effect-cancellation-request-v1.schema.json create mode 100644 src/determa/state/data/effect-cancellation-response-v1.schema.json create mode 100644 src/determa/state/data/effect-result-request-v1.schema.json create mode 100644 src/determa/state/data/effect-result-response-v1.schema.json create mode 100644 src/determa/state/data/host-effect-claim-v1.schema.json create mode 100644 src/determa/state/data/host-effect-journal-v1.schema.json diff --git a/conformance/authority_adapter.py b/conformance/authority_adapter.py index 3cd6d9b..5a50fa6 100644 --- a/conformance/authority_adapter.py +++ b/conformance/authority_adapter.py @@ -22,17 +22,25 @@ configure_bundled_sqlite_authority, ) from determa.state.extensions import bundled_extension_registry -from determa.state.wire import hash_value +from determa.state.effects import SQLiteCommittedEffectHost +from determa.state.wire import MemoryArtifactResolver, hash_value _PATH = Path(tempfile.gettempdir()) / "determa-python-local-authority-profile.sqlite" _INITIAL_BINDING: str | None = None +_WORKER_MODE = "--worker" in sys.argv[1:] def _profile() -> dict[str, Any]: authority = _seed(_baseline_ledger()) registry = bundled_extension_registry(include_postgresql=False) configured, authority, store = configure_bundled_sqlite_authority( - registry, authority.path, "scope-42", "owner-1", "2" + registry, + authority.path, + "scope-42", + "owner-1", + "2", + worker_fencing=_WORKER_MODE, + worker_lease_nanoseconds=1, ) report = authority.profile_report("scope-42", "owner-1", store, registry, configured) installed = authority.profile_descriptor(store, registry, configured)["installation_evidence"] @@ -47,7 +55,20 @@ def _profile() -> dict[str, Any]: "freeze_waits_for_writer", "freeze_after_drain", "incomplete_frozen_inventory_refuses_freeze", - ], + ] + + ( + [ + "fence_worker_allocates_new_claim", + "dispatch_at_expiry", + "result_at_expiry", + "clock_unavailable", + "old_epoch", + "old_attempt", + "principal_mismatch", + ] + if _WORKER_MODE + else [] + ), }, } @@ -55,7 +76,13 @@ def _profile() -> dict[str, Any]: def _binding_live() -> tuple[str, SQLiteLocalAuthority]: registry = bundled_extension_registry(include_postgresql=False) configured, authority, store = configure_bundled_sqlite_authority( - registry, _PATH, "scope-42", "owner-1", "2" + registry, + _PATH, + "scope-42", + "owner-1", + "2", + worker_fencing=_WORKER_MODE, + worker_lease_nanoseconds=1, ) descriptor = authority.profile_descriptor(store, registry, configured) return hash_value( @@ -67,6 +94,7 @@ def _binding_live() -> tuple[str, SQLiteLocalAuthority]: descriptor["source_binding_digest"], descriptor["destination_binding_digest"], descriptor["extension_report"], + descriptor["required_participants"], ] ), authority @@ -111,9 +139,11 @@ def _seed(ledger: dict[str, Any]) -> SQLiteLocalAuthority: # removes an allocated scope marker from a live authority domain. for suffix in ("", "-wal", "-shm"): (_PATH.parent / (_PATH.name + suffix)).unlink(missing_ok=True) - authority = SQLiteLocalAuthority(_PATH) + authority = SQLiteLocalAuthority(_PATH, worker_fencing=_WORKER_MODE, worker_lease_nanoseconds=1) authority.setup_schema() AuthoritySQLiteExecutionStore(authority, "scope-42", "owner-1", "2").setup_schema() + if _WORKER_MODE: + SQLiteCommittedEffectHost(_PATH, MemoryArtifactResolver(), {}, lambda *_: {}).setup_schema() if not authority._insert_ledger(_baseline_ledger()): raise RuntimeError("fixture scope already allocated") _INITIAL_BINDING, authority = _binding_live() @@ -174,7 +204,9 @@ def start(session: str, step: int, barrier: str) -> None: held = context.Event() def disconnected_writer() -> None: - local = SQLiteLocalAuthority(_PATH) + local = SQLiteLocalAuthority( + _PATH, worker_fencing=_WORKER_MODE, worker_lease_nanoseconds=1 + ) def hold_transaction() -> None: held.set() @@ -357,7 +389,25 @@ def run(payload: dict[str, Any]) -> dict[str, Any]: if kind == "profile": return _common_rule(payload["configured_facts"], set()) if kind == "worker_claim_check": - raise NotImplementedError("worker claim checks unavailable") + if not _WORKER_MODE: + raise NotImplementedError("worker claim checks unavailable") + item = payload["input"] + authority = _seed(item["ledger_before"]) + dispatched: list[str] = [] + accepted = authority.check_worker_claim( + item["claim"], + item["authenticated_principal"], + item["trusted_clock_now"], + phase=item["phase"], + on_dispatch=lambda: dispatched.append("called"), + ) + return { + "binding": _binding(), + "accepted": accepted, + "ledger_after": authority.inspect("scope-42"), + "host_mutation_count": 0, + "external_dispatch_count": len(dispatched), + } raise ValueError(f"unknown driver kind: {kind}") diff --git a/conformance/worker_authority_adapter.py b/conformance/worker_authority_adapter.py new file mode 100644 index 0000000..d66fbe1 --- /dev/null +++ b/conformance/worker_authority_adapter.py @@ -0,0 +1,17 @@ +"""Use the installed worker-capable SQLite authority for §18 and §19 probes.""" + +from __future__ import annotations + +import sys + +from conformance import authority_adapter + + +def main() -> None: + authority_adapter._WORKER_MODE = True + payload = authority_adapter._parse(sys.stdin.read()) + print(authority_adapter._compact(authority_adapter.run(payload))) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index b77844f..65bd518 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,12 @@ packages = ["src/determa"] "src/determa/state/data/inspection-v1.schema.json" = "determa/state/data/inspection-v1.schema.json" "src/determa/state/data/host-authority-operation-v1.schema.json" = "determa/state/data/host-authority-operation-v1.schema.json" "src/determa/state/data/host-authority-profile-report-v1.schema.json" = "determa/state/data/host-authority-profile-report-v1.schema.json" +"src/determa/state/data/host-effect-journal-v1.schema.json" = "determa/state/data/host-effect-journal-v1.schema.json" +"src/determa/state/data/host-effect-claim-v1.schema.json" = "determa/state/data/host-effect-claim-v1.schema.json" +"src/determa/state/data/effect-result-request-v1.schema.json" = "determa/state/data/effect-result-request-v1.schema.json" +"src/determa/state/data/effect-result-response-v1.schema.json" = "determa/state/data/effect-result-response-v1.schema.json" +"src/determa/state/data/effect-cancellation-request-v1.schema.json" = "determa/state/data/effect-cancellation-request-v1.schema.json" +"src/determa/state/data/effect-cancellation-response-v1.schema.json" = "determa/state/data/effect-cancellation-response-v1.schema.json" [tool.ruff] line-length = 100 diff --git a/src/determa/state/authority.py b/src/determa/state/authority.py index 05c94ca..112603e 100644 --- a/src/determa/state/authority.py +++ b/src/determa/state/authority.py @@ -97,6 +97,7 @@ def _result( code: str | None, *, early: bool = False, + claim: Mapping[str, Any] | None = None, ) -> dict[str, Any]: response = { "interface": _INTERFACE, @@ -114,7 +115,7 @@ def _result( "state": None if early or ledger is None else ledger["state"], "evidence_digest": None, "error_code": code, - "claim": None, + "claim": None if claim is None else copy.deepcopy(dict(claim)), } if not code and request is not None: response["evidence_digest"] = hash_value( @@ -223,11 +224,26 @@ def _inventory(ledger: Mapping[str, Any]) -> list[dict[str, str]]: class SQLiteLocalAuthority: """Single-database guarded authority; no worker or relocation guarantee.""" - def __init__(self, path: str | Path, *, timeout: float = 30.0) -> None: + def __init__( + self, + path: str | Path, + *, + timeout: float = 30.0, + worker_fencing: bool = False, + worker_lease_nanoseconds: int = 1_000_000_000, + ) -> None: if not str(path) or str(path) == ":memory:" or timeout <= 0: raise ValueError("a persistent SQLite file and positive timeout are required") self.path = str(Path(path).resolve()) self.timeout = timeout + if ( + type(worker_fencing) is not bool + or type(worker_lease_nanoseconds) is not int + or worker_lease_nanoseconds <= 0 + ): + raise ValueError("invalid worker authority configuration") + self.worker_fencing = worker_fencing + self.worker_lease_nanoseconds = worker_lease_nanoseconds def _connect(self) -> sqlite3.Connection: connection = sqlite3.connect(self.path, timeout=self.timeout, isolation_level=None) @@ -299,17 +315,50 @@ def profile_descriptor( "consistent_scope_inventory", }: raise ValueError("host_capability_mismatch") - closure = _authority_closure() - configuration = _compact( - { - "sqlite_path": self.path, - "journal_mode": "WAL", - "synchronous": "FULL", - "topology": "single-sqlite-database", - "replay_retention": execution_store.replay_retention, - "outbox_retention": execution_store.outbox_retention, - } - ).encode() + authority_closure = _authority_closure() + configuration_values = { + "sqlite_path": self.path, + "journal_mode": "WAL", + "synchronous": "FULL", + "topology": "single-sqlite-database", + "replay_retention": execution_store.replay_retention, + "outbox_retention": execution_store.outbox_retention, + } + if self.worker_fencing: + configuration_values["worker_fencing"] = True + configuration_values["worker_lease_nanoseconds"] = self.worker_lease_nanoseconds + configuration = _compact(configuration_values).encode() + participants = [] + participant_installations = [] + if self.worker_fencing: + with self._connect() as connection: + columns = connection.execute( + "PRAGMA table_info(determa_committed_effects)" + ).fetchall() + if [row[1] for row in columns] != ["root_instance_id", "document"]: + raise ValueError("worker journal is not installed") + for role, name, source in ( + ("journal", "journal-1", Path(__file__).with_name("effects.py")), + ("worker", "worker-1", Path(__file__)), + ): + closure = source.read_bytes() + participant = { + "role": role, + "provider_reference": { + "identifier": f"reference.local-{role}", + "version": __version__, + "content_digest": "sha256:" + hashlib.sha256(closure).hexdigest(), + }, + "instance_id": name, + } + participants.append(participant) + participant_installations.append( + { + "participant": copy.deepcopy(participant), + "closure_bytes_base64": base64.b64encode(closure).decode(), + "observed_health": "healthy", + } + ) return { "extension_report": extension_report, "authority_storage_boundary": "local-sqlite-authority-db", @@ -321,18 +370,18 @@ def profile_descriptor( }, "source_binding_digest": hash_value(["determa-local-authority-source-1", self.path]), "destination_binding_digest": None, - "required_participants": [], + "required_participants": participants, "guarantees": { "guarded_local_writes": True, - "worker_fencing": False, + "worker_fencing": self.worker_fencing, "complete_scope_inventory": True, "safe_relocation": False, }, "installation_evidence": { - "closure_bytes_base64": base64.b64encode(closure).decode(), + "closure_bytes_base64": base64.b64encode(authority_closure).decode(), "configuration_bytes_base64": base64.b64encode(configuration).decode(), "observed_health": "healthy", - "participant_installations": [], + "participant_installations": participant_installations, }, } @@ -440,6 +489,67 @@ def inspect(self, scope_identity: str) -> dict[str, Any] | None: ).fetchone() return None if row is None else _parse(row[0]) + def check_worker_claim( + self, + claim: Mapping[str, Any], + authenticated_principal: str, + trusted_clock_now: str | None, + *, + phase: str, + on_dispatch: Callable[[], None] | None = None, + ) -> bool: + """Check the stored claim under the scope guard before dispatch or result work.""" + if not self.worker_fencing or phase not in {"dispatch", "result"}: + return False + if ( + type(trusted_clock_now) is not str + or re.fullmatch(r"(?:0|-?[1-9][0-9]*)", trusted_clock_now) is None + ): + return False + now = int(trusted_clock_now) + if not -(2**63) <= now < 2**63: + return False + with self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + "SELECT ledger FROM determa_scope_authority WHERE scope_identity = ?", + (claim.get("scope_identity"),), + ).fetchone() + if row is None: + return False + ledger = _parse(row[0]) + stored = next( + ( + item + for item in ledger["active_claims"] + if item["work_identity"] == claim.get("work_identity") + ), + None, + ) + journal = next( + ( + item + for item in ledger["journal_entries"] + if item["work_identity"] == claim.get("work_identity") + ), + None, + ) + if ( + stored is None + or journal is None + or stored != dict(claim) + or ledger["state"] != "active" + or stored["scope_authority_epoch"] != ledger["authority_epoch"] + or stored["attempt_fence"] != journal["attempt_fence"] + or stored["worker_principal"] != authenticated_principal + or stored["state"] != "active" + or now >= int(stored["expires_at"]) + ): + return False + if phase == "dispatch" and on_dispatch is not None: + on_dispatch() + return True + def perform( self, request_bytes: str, @@ -531,7 +641,7 @@ def _perform( "scope_fence_unproven" if invalid_proof else "host_capability_mismatch", ) ) - if operation == "fence_worker": + if operation == "fence_worker" and not self.worker_fencing: return _compact(_result(request, ledger, "host_capability_mismatch")) if isinstance(fault, dict) and fault.get("epoch_check_separate_from_commit"): return _compact(_result(request, ledger, "host_capability_mismatch")) @@ -553,11 +663,62 @@ def _perform( return _compact(_result(request, ledger, "stale_scope_authority")) if request["expected_scope_generation"] != ledger["scope_generation"]: return _compact(_result(request, ledger, "scope_generation_conflict")) - if ( - ledger["state"] != "active" - or invocation["authenticated_principal"] != ledger["owner_principal"] + if ledger["state"] != "active" or ( + operation != "fence_worker" + and invocation["authenticated_principal"] != ledger["owner_principal"] ): return _compact(_result(request, ledger, "stale_scope_authority")) + new_claim = None + if operation == "fence_worker": + arguments = request["arguments"] + principal = invocation["authenticated_principal"] + if arguments["expected_worker_principal"] != principal: + return _compact(_result(request, ledger, "worker_principal_mismatch")) + now = invocation.get("trusted_clock_now") + if ( + invocation.get("clock_available") is not True + or type(now) is not str + or re.fullmatch(r"(?:0|-?[1-9][0-9]*)", now) is None + or not -(2**63) <= int(now) < 2**63 + or int(now) + self.worker_lease_nanoseconds >= 2**63 + ): + return _compact(_result(request, ledger, "host_capability_mismatch")) + entry = next( + ( + item + for item in ledger["journal_entries"] + if item["work_identity"] == arguments["work_identity"] + ), + None, + ) + if ( + entry is None + or arguments["root_instance_id"] not in ledger["roots"] + or entry["attempt_fence"] != arguments["expected_attempt_fence"] + ): + return _compact(_result(request, ledger, "stale_attempt_fence")) + if invocation.get("assigned_worker_principal") != principal: + return _compact(_result(request, ledger, "worker_principal_mismatch")) + fence = str(int(entry["attempt_fence"]) + 1) + new_claim = { + "scope_identity": scope, + "root_instance_id": arguments["root_instance_id"], + "work_kind": "effect", + "work_identity": arguments["work_identity"], + "operation_token": arguments["operation_token"], + "scope_authority_epoch": ledger["authority_epoch"], + "attempt_fence": fence, + "worker_principal": principal, + "expires_at": str(int(now) + self.worker_lease_nanoseconds), + "state": "active", + } + entry["attempt_fence"] = fence + ledger["active_claims"] = [ + item + for item in ledger["active_claims"] + if item["work_identity"] != arguments["work_identity"] + ] + ledger["active_claims"].append(new_claim) if operation == "guarded_commit": if ( native_mutation_bytes is None @@ -571,7 +732,7 @@ def _perform( if fault == "omit_receipt_from_inventory": return _compact(_result(request, ledger, "scope_fence_unproven")) if ( - ledger["required_participant_records"] + (ledger["required_participant_records"] and not self.worker_fencing) or ledger["journal_entries"] or ledger["active_claims"] ): @@ -580,7 +741,7 @@ def _perform( if operation == "freeze_scope": ledger["state"] = "frozen" ledger["active_claims"] = [] - response = _result(request, ledger, None) + response = _result(request, ledger, None, claim=new_claim) ledger["receipts"].append( { "operation_id": request["operation_id"], @@ -793,7 +954,7 @@ def bundled_authority_descriptor() -> dict[str, Any]: class _BundledAuthorityProvider: def validate_configuration(self, configuration: Mapping[str, Any]) -> dict[str, Any]: - if set(configuration) != { + required = { "instance_id", "path", "scope_identity", @@ -801,13 +962,30 @@ def validate_configuration(self, configuration: Mapping[str, Any]) -> dict[str, "authority_epoch", "replay_retention", "outbox_retention", - } or any( - type(configuration[key]) is not str or not configuration[key] for key in configuration + } + worker_keys = {"worker_fencing", "worker_lease_nanoseconds"} + if ( + set(configuration) not in (required, required | worker_keys) + or any( + type(configuration[key]) is not str or not configuration[key] for key in required + ) + or ( + worker_keys <= set(configuration) + and ( + configuration["worker_fencing"] is not True + or type(configuration["worker_lease_nanoseconds"]) is not int + or configuration["worker_lease_nanoseconds"] <= 0 + ) + ) ): raise ValueError("invalid_extension_configuration") if configuration["instance_id"] != "local-authority": raise ValueError("invalid_extension_configuration") - authority = SQLiteLocalAuthority(configuration["path"]) + authority = SQLiteLocalAuthority( + configuration["path"], + worker_fencing=configuration.get("worker_fencing", False), + worker_lease_nanoseconds=configuration.get("worker_lease_nanoseconds", 1_000_000_000), + ) if authority.path != configuration["path"]: raise ValueError("invalid_extension_configuration") store = AuthoritySQLiteExecutionStore( @@ -841,6 +1019,13 @@ def health(self, instance: Any) -> str: or ledger["state"] not in ("active", "frozen") ): return "unavailable" + if authority.worker_fencing: + with authority._connect() as connection: + columns = connection.execute( + "PRAGMA table_info(determa_committed_effects)" + ).fetchall() + if [row[1] for row in columns] != ["root_instance_id", "document"]: + return "unavailable" except (ValueError, sqlite3.Error, KeyError): return "unavailable" return "healthy" @@ -859,9 +1044,11 @@ def configure_bundled_sqlite_authority( *, replay_retention: str = "bounded", outbox_retention: str = "none", + worker_fencing: bool = False, + worker_lease_nanoseconds: int = 1_000_000_000, ) -> tuple[ConfiguredExtension, SQLiteLocalAuthority, AuthoritySQLiteExecutionStore]: """Configure and return the exact authority/store instance used for host operations.""" - configuration = { + configuration: dict[str, Any] = { "instance_id": "local-authority", "path": str(Path(path).resolve()), "scope_identity": scope_identity, @@ -870,6 +1057,9 @@ def configure_bundled_sqlite_authority( "replay_retention": replay_retention, "outbox_retention": outbox_retention, } + if worker_fencing: + configuration["worker_fencing"] = True + configuration["worker_lease_nanoseconds"] = worker_lease_nanoseconds configured = registry.validate_configuration(bundled_authority_descriptor(), configuration) _, _, instance, _ = registry._bound(configured) return configured, instance["authority"], instance["store"] diff --git a/src/determa/state/data/effect-cancellation-request-v1.schema.json b/src/determa/state/data/effect-cancellation-request-v1.schema.json new file mode 100644 index 0000000..7181b5f --- /dev/null +++ b/src/determa/state/data/effect-cancellation-request-v1.schema.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://determa.dev/state/schema/effect-cancellation-request-v1.schema.json", + "title": "Determa State version 1 effect cancellation request v1", + "type": "object", + "required": [ + "operation_id", + "effect_id", + "reason", + "payload" + ], + "additionalProperties": false, + "properties": { + "operation_id": { + "type": "string", + "minLength": 1 + }, + "effect_id": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "reason": { + "type": "string", + "minLength": 1 + }, + "payload": { + "$ref": "aggregate-state-v1.schema.json#/$defs/typedMap" + } + } +} diff --git a/src/determa/state/data/effect-cancellation-response-v1.schema.json b/src/determa/state/data/effect-cancellation-response-v1.schema.json new file mode 100644 index 0000000..ce55864 --- /dev/null +++ b/src/determa/state/data/effect-cancellation-response-v1.schema.json @@ -0,0 +1,186 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://determa.dev/state/schema/effect-cancellation-response-v1.schema.json", + "title": "Determa State version 1 effect cancellation response v1", + "oneOf": [ + { + "type": "object", + "required": [ + "status", + "operation_id", + "effect_id", + "cancellation", + "outcome", + "result_event_id", + "journal_revision", + "error_code" + ], + "additionalProperties": false, + "properties": { + "status": { + "const": "committed" + }, + "operation_id": { + "type": "string", + "minLength": 1 + }, + "effect_id": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "cancellation": { + "type": "object", + "required": [ + "operation_id", + "reason", + "state" + ], + "additionalProperties": false, + "properties": { + "operation_id": { + "type": "string", + "minLength": 1 + }, + "reason": { + "type": "string", + "minLength": 1 + }, + "state": { + "enum": [ + "prevented_start", + "too_late" + ] + } + } + }, + "outcome": { + "$ref": "host-effect-journal-v1.schema.json#/$defs/outcome" + }, + "result_event_id": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "journal_revision": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "error_code": { + "type": "null" + } + } + }, + { + "type": "object", + "required": [ + "status", + "operation_id", + "effect_id", + "cancellation", + "outcome", + "result_event_id", + "journal_revision", + "error_code" + ], + "additionalProperties": false, + "properties": { + "status": { + "const": "reconciliation_required" + }, + "operation_id": { + "type": "string", + "minLength": 1 + }, + "effect_id": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "cancellation": { + "type": "object", + "required": [ + "operation_id", + "reason", + "state" + ], + "additionalProperties": false, + "properties": { + "operation_id": { + "type": "string", + "minLength": 1 + }, + "reason": { + "type": "string", + "minLength": 1 + }, + "state": { + "const": "reconciliation_required" + } + } + }, + "outcome": { + "type": "null" + }, + "result_event_id": { + "type": "null" + }, + "journal_revision": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "error_code": { + "type": "null" + } + } + }, + { + "type": "object", + "required": [ + "status", + "operation_id", + "effect_id", + "cancellation", + "outcome", + "result_event_id", + "journal_revision", + "error_code" + ], + "additionalProperties": false, + "properties": { + "status": { + "const": "rejected" + }, + "operation_id": { + "type": "string", + "minLength": 1 + }, + "effect_id": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "cancellation": { + "type": "null" + }, + "outcome": { + "type": "null" + }, + "result_event_id": { + "type": "null" + }, + "journal_revision": { + "type": "null" + }, + "error_code": { + "enum": [ + "unauthorized_scope", + "scope_not_active", + "stale_scope_authority", + "host_capability_mismatch", + "effect_not_outstanding", + "effect_result_conflict", + "operation_id_conflict", + "invalid_host_request" + ] + } + } + } + ] +} diff --git a/src/determa/state/data/effect-result-request-v1.schema.json b/src/determa/state/data/effect-result-request-v1.schema.json new file mode 100644 index 0000000..d0455c1 --- /dev/null +++ b/src/determa/state/data/effect-result-request-v1.schema.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://determa.dev/state/schema/effect-result-request-v1.schema.json", + "title": "Determa State version 1 effect result request", + "$ref": "#/$defs/request", + "$defs": { + "request": { + "type": "object", + "required": [ + "effect_id", + "operation_token", + "attempt_fence", + "outcome_kind", + "payload" + ], + "additionalProperties": false, + "properties": { + "effect_id": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "operation_token": { + "type": "string", + "minLength": 1 + }, + "attempt_fence": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "outcome_kind": { + "enum": [ + "succeeded", + "domain_rejected", + "retryable_failure", + "terminal_failure", + "cancelled", + "ambiguous" + ] + }, + "payload": { + "$ref": "aggregate-state-v1.schema.json#/$defs/typedMap" + } + } + } + } +} diff --git a/src/determa/state/data/effect-result-response-v1.schema.json b/src/determa/state/data/effect-result-response-v1.schema.json new file mode 100644 index 0000000..6f92baf --- /dev/null +++ b/src/determa/state/data/effect-result-response-v1.schema.json @@ -0,0 +1,173 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://determa.dev/state/schema/effect-result-response-v1.schema.json", + "title": "Determa State version 1 effect result response", + "oneOf": [ + { + "type": "object", + "required": [ + "status", + "effect_id", + "attempt_fence", + "attempt_report", + "outcome", + "result_event_id", + "admission_receipt", + "checkpoint_revision", + "journal_revision", + "error_code" + ], + "additionalProperties": false, + "properties": { + "status": { + "const": "committed" + }, + "effect_id": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "attempt_fence": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "attempt_report": { + "type": "null" + }, + "outcome": { + "$ref": "host-effect-journal-v1.schema.json#/$defs/outcome" + }, + "result_event_id": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "admission_receipt": { + "$ref": "execution-checkpoint-v1.schema.json#/$defs/acceptanceReceipt" + }, + "checkpoint_revision": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "journal_revision": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "error_code": { + "type": "null" + } + } + }, + { + "type": "object", + "required": [ + "status", + "effect_id", + "attempt_fence", + "attempt_report", + "outcome", + "result_event_id", + "admission_receipt", + "checkpoint_revision", + "journal_revision", + "error_code" + ], + "additionalProperties": false, + "properties": { + "status": { + "const": "report_recorded" + }, + "effect_id": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "attempt_fence": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "attempt_report": { + "$ref": "host-effect-journal-v1.schema.json#/$defs/attemptRecord" + }, + "outcome": { + "type": "null" + }, + "result_event_id": { + "type": "null" + }, + "admission_receipt": { + "type": "null" + }, + "checkpoint_revision": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "journal_revision": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "error_code": { + "type": "null" + } + } + }, + { + "type": "object", + "required": [ + "status", + "effect_id", + "attempt_fence", + "attempt_report", + "outcome", + "result_event_id", + "admission_receipt", + "checkpoint_revision", + "journal_revision", + "error_code" + ], + "additionalProperties": false, + "properties": { + "status": { + "const": "rejected" + }, + "effect_id": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "attempt_fence": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "attempt_report": { + "type": "null" + }, + "outcome": { + "type": "null" + }, + "result_event_id": { + "type": "null" + }, + "admission_receipt": { + "type": "null" + }, + "checkpoint_revision": { + "type": "null" + }, + "journal_revision": { + "type": "null" + }, + "error_code": { + "enum": [ + "unauthorized_scope", + "scope_not_active", + "stale_scope_authority", + "host_capability_mismatch", + "effect_not_outstanding", + "effect_result_conflict", + "stale_attempt_fence", + "runtime_provider_output_invalid", + "portable_value_required", + "invalid_host_request" + ] + } + } + } + ] +} diff --git a/src/determa/state/data/host-effect-claim-v1.schema.json b/src/determa/state/data/host-effect-claim-v1.schema.json new file mode 100644 index 0000000..07b8252 --- /dev/null +++ b/src/determa/state/data/host-effect-claim-v1.schema.json @@ -0,0 +1,69 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://determa.dev/state/schema/host-effect-claim-v1.schema.json", + "title": "Determa State version 1 host effect claim", + "$ref": "#/$defs/claim", + "$defs": { + "claim": { + "type": "object", + "required": [ + "scope_identity", + "root_instance_id", + "work_kind", + "work_identity", + "operation_token", + "scope_authority_epoch", + "attempt_fence", + "worker_principal", + "expires_at", + "state" + ], + "additionalProperties": false, + "properties": { + "scope_identity": { + "type": "string", + "minLength": 1 + }, + "root_instance_id": { + "type": "string", + "minLength": 1 + }, + "work_kind": { + "const": "effect" + }, + "work_identity": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "operation_token": { + "type": "string", + "minLength": 1 + }, + "scope_authority_epoch": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "attempt_fence": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "worker_principal": { + "type": "string", + "minLength": 1 + }, + "expires_at": { + "type": "string", + "pattern": "^(0|-?[1-9][0-9]*)$", + "description": "Canonical signed Unix-epoch nanoseconds; enforce signed 64-bit bounds semantically." + }, + "state": { + "enum": [ + "active", + "revoked", + "expired" + ] + } + } + } + } +} diff --git a/src/determa/state/data/host-effect-journal-v1.schema.json b/src/determa/state/data/host-effect-journal-v1.schema.json new file mode 100644 index 0000000..087a9d7 --- /dev/null +++ b/src/determa/state/data/host-effect-journal-v1.schema.json @@ -0,0 +1,393 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://determa.dev/state/schema/host-effect-journal-v1.schema.json", + "title": "Determa State version 1 host effect journal", + "description": "Closed host-owned native-effect execution evidence; separate from portable checkpoint.", + "$ref": "#/$defs/hostEffectJournal", + "$defs": { + "sha256": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "nonEmptyString": { + "type": "string", + "minLength": 1 + }, + "canonicalDecimal": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "portableValue": { + "$ref": "aggregate-state-v1.schema.json#/$defs/typedValue" + }, + "portablePayload": { + "$ref": "aggregate-state-v1.schema.json#/$defs/typedMap" + }, + "handlerReference": { + "$ref": "provider-reference-v1.schema.json" + }, + "operationTokenLocation": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "kind" + ], + "additionalProperties": false, + "properties": { + "kind": { + "const": "correlation_id" + } + } + }, + { + "type": "object", + "required": [ + "kind", + "pointer" + ], + "additionalProperties": false, + "properties": { + "kind": { + "const": "payload" + }, + "pointer": { + "type": "string", + "pattern": "^(|(/([^~]|~0|~1)*)*)$" + } + } + } + ] + }, + "resultMapping": { + "type": "object", + "required": [ + "outcome_kind", + "event", + "result_slot", + "operation_token_location" + ], + "additionalProperties": false, + "properties": { + "outcome_kind": { + "enum": [ + "succeeded", + "domain_rejected", + "terminal_failure", + "cancelled" + ] + }, + "event": { + "$ref": "#/$defs/nonEmptyString" + }, + "result_slot": { + "$ref": "#/$defs/nonEmptyString" + }, + "operation_token_location": { + "$ref": "#/$defs/operationTokenLocation" + } + } + }, + "target": { + "type": "object", + "required": [ + "root_instance_id", + "runtime_id", + "runtime_incarnation" + ], + "additionalProperties": false, + "properties": { + "root_instance_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "runtime_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "runtime_incarnation": { + "$ref": "aggregate-state-v1.schema.json#/$defs/identityOrigin" + } + } + }, + "attemptRecord": { + "type": "object", + "required": [ + "attempt_fence", + "report_kind", + "report_digest", + "reason" + ], + "additionalProperties": false, + "properties": { + "attempt_fence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "report_kind": { + "enum": [ + "succeeded", + "domain_rejected", + "retryable_failure", + "terminal_failure", + "cancelled", + "ambiguous" + ] + }, + "report_digest": { + "$ref": "#/$defs/sha256" + }, + "reason": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/nonEmptyString" + } + ] + } + } + }, + "outcome": { + "type": "object", + "required": [ + "kind", + "payload", + "digest", + "attempt_fence" + ], + "additionalProperties": false, + "properties": { + "kind": { + "enum": [ + "succeeded", + "domain_rejected", + "terminal_failure", + "cancelled" + ] + }, + "payload": { + "$ref": "aggregate-state-v1.schema.json#/$defs/typedMap" + }, + "digest": { + "$ref": "#/$defs/sha256" + }, + "attempt_fence": { + "$ref": "#/$defs/canonicalDecimal" + } + } + }, + "cancellation": { + "type": "object", + "required": [ + "operation_id", + "reason", + "state" + ], + "additionalProperties": false, + "properties": { + "operation_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "reason": { + "$ref": "#/$defs/nonEmptyString" + }, + "state": { + "enum": [ + "requested", + "prevented_start", + "too_late", + "reconciliation_required" + ] + } + } + }, + "effectRecord": { + "type": "object", + "required": [ + "effect_id", + "operation_token", + "intent_digest", + "handler_reference", + "destination_binding_digest", + "route_configuration_generation", + "result_mapping", + "target", + "idempotency_policy", + "attempt_fence", + "attempt_records", + "invocation_state", + "outcome", + "result_event_id", + "admission_receipt", + "cancellation" + ], + "additionalProperties": false, + "properties": { + "effect_id": { + "$ref": "#/$defs/sha256" + }, + "operation_token": { + "$ref": "#/$defs/nonEmptyString" + }, + "intent_digest": { + "$ref": "#/$defs/sha256" + }, + "handler_reference": { + "$ref": "#/$defs/handlerReference" + }, + "destination_binding_digest": { + "$ref": "#/$defs/sha256" + }, + "route_configuration_generation": { + "$ref": "#/$defs/canonicalDecimal" + }, + "result_mapping": { + "type": "array", + "items": { + "$ref": "#/$defs/resultMapping" + } + }, + "target": { + "$ref": "#/$defs/target" + }, + "idempotency_policy": { + "enum": [ + "destination_deduplicates", + "reconcile_before_retry" + ] + }, + "attempt_fence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "attempt_records": { + "type": "array", + "items": { + "$ref": "#/$defs/attemptRecord" + } + }, + "invocation_state": { + "enum": [ + "unclaimed", + "leased", + "ambiguous", + "outcome_recorded", + "result_admitted", + "closed" + ] + }, + "outcome": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/outcome" + } + ] + }, + "result_event_id": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/sha256" + } + ] + }, + "admission_receipt": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "execution-checkpoint-v1.schema.json#/$defs/acceptanceReceipt" + } + ] + }, + "cancellation": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/cancellation" + } + ] + } + } + }, + "operationResponseReference": { + "type": "object", + "required": [ + "operation_id", + "response_digest" + ], + "additionalProperties": false, + "properties": { + "operation_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "response_digest": { + "$ref": "#/$defs/sha256" + } + } + }, + "hostEffectJournal": { + "type": "object", + "required": [ + "host_effect_journal_format", + "host_effect_journal_schema_version", + "scope_identity", + "root_instance_id", + "checkpoint_revision", + "checkpoint_digest", + "journal_revision", + "effect_records", + "operation_response_references", + "host_effect_journal_digest" + ], + "additionalProperties": false, + "properties": { + "host_effect_journal_format": { + "const": "determa.host_effect_journal" + }, + "host_effect_journal_schema_version": { + "const": 1 + }, + "scope_identity": { + "$ref": "#/$defs/nonEmptyString" + }, + "root_instance_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "checkpoint_revision": { + "$ref": "#/$defs/canonicalDecimal" + }, + "checkpoint_digest": { + "$ref": "#/$defs/sha256" + }, + "journal_revision": { + "$ref": "#/$defs/canonicalDecimal" + }, + "effect_records": { + "type": "array", + "items": { + "$ref": "#/$defs/effectRecord" + } + }, + "operation_response_references": { + "type": "array", + "items": { + "$ref": "#/$defs/operationResponseReference" + } + }, + "host_effect_journal_digest": { + "$ref": "#/$defs/sha256" + } + } + } + } +} From 20eda06f0d2bca51992f377b4499986b24b724b9 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 05:22:10 +0000 Subject: [PATCH 03/30] Reconstruct missing effect tails and replay retained corrections Preserve both original retained20000-character prefixes, reconstruct only missing tails from approved contracts and retained source observations, and reapply every complete effects implementation/adapter correction patch. Complete-file original bytes are not claimed recovered. Lint, formatting and mypy pass. Functional validation, authority integration fixes and independent review remain unfinished; draft only. --- conformance/effects_adapter.py | 677 +++++++++++++++++++ src/determa/state/effects.py | 1123 ++++++++++++++++++++++++++++++++ 2 files changed, 1800 insertions(+) create mode 100644 conformance/effects_adapter.py create mode 100644 src/determa/state/effects.py diff --git a/conformance/effects_adapter.py b/conformance/effects_adapter.py new file mode 100644 index 0000000..1ae58df --- /dev/null +++ b/conformance/effects_adapter.py @@ -0,0 +1,677 @@ +#!/usr/bin/env python3 +"""Production SQLite and installed native-handler bridge for the §19 profile.""" + +from __future__ import annotations + +import base64 +import copy +import hashlib +import importlib.util +import json +import os +import sqlite3 +import subprocess +import sys +import time +import uuid +from pathlib import Path +from typing import Any + +from conformance.authority_adapter import _PATH +from determa.state import load_bundle +from determa.state.authority import ( + AuthoritySQLiteExecutionStore, + SQLiteLocalAuthority, + _compact, + configure_bundled_sqlite_authority, +) +from determa.state.effects import ( + EffectError, + SQLiteCommittedEffectHost, + _result_response, +) +from determa.state.extensions import bundled_extension_registry +from determa.state.wire import ( + MemoryArtifactResolver, + canonical_bytes, + decoded_typed_value, +) + +_CASE = ( + Path(os.environ["DETERMA_CONFORMANCE_DIR"]) + / "conformance/profiles/committed-native-effects/effect-01-result" +) +_DESTINATION_PATH = _PATH.with_name("determa-effect-fake-destination.sqlite") +_PROOFS_PATH = _PATH.with_name("determa-effect-native-proofs.json") +_AUTHORITY_SCOPE = "effect-scope-1" +_ROOT = "effect-root-1" +_HANDLER_SOURCE = "handler/test_handler.py" +_CONTROL_ROOT = _PATH.with_name("determa-effect-control") + + +def _await_file(path: Path, seconds: float = 15) -> None: + deadline = time.monotonic() + seconds + while not path.exists(): + if time.monotonic() >= deadline: + raise TimeoutError(f"native control barrier was not reached: {path.name}") + time.sleep(0.02) + + +def _sha(source: bytes) -> str: + return "sha256:" + hashlib.sha256(source).hexdigest() + + +def _handler_closure() -> bytes: + result = bytearray(b"determa-effect-handler-closure-1\0") + for path in sorted((_CASE / "handler").glob("test_handler.*")): + name = str(path.relative_to(_CASE)).encode() + source = path.read_bytes() + result += len(name).to_bytes(8, "big") + name + result += len(source).to_bytes(8, "big") + source + return bytes(result) + + +def _destination_configuration() -> bytes: + return canonical_bytes(json.loads((_CASE / "data/destination-configuration.json").read_bytes())) + + +def _route(checkpoint: dict[str, Any], generation: str) -> dict[str, Any]: + runtime = checkpoint["root_record"]["aggregate_state"]["runtimes"][0] + return { + "generation": generation, + "authority_epoch": "3", + "handler_reference": { + "identifier": "conformance.native-effect-handler", + "version": "1.0.0", + "content_digest": _sha(_handler_closure()), + }, + "destination_binding_digest": _sha(_destination_configuration()), + "result_mapping": [ + { + "outcome_kind": "succeeded", + "event": "native_succeeded", + "result_slot": "success", + "operation_token_location": {"kind": "correlation_id"}, + }, + { + "outcome_kind": "cancelled", + "event": "native_cancelled", + "result_slot": "cancelled", + "operation_token_location": {"kind": "correlation_id"}, + }, + ], + "target": { + "root_instance_id": checkpoint["root_instance_id"], + "runtime_id": runtime["runtime_id"], + "runtime_incarnation": copy.deepcopy(runtime["identity_origin"]), + }, + "idempotency_policy": "destination_deduplicates", + } + + +def _authority_ledger() -> dict[str, Any]: + return { + "allocated_scope_identities": [_AUTHORITY_SCOPE], + "scope_identity": _AUTHORITY_SCOPE, + "owner_principal": "owner-1", + "authority_epoch": "3", + "scope_generation": "4", + "state": "active", + "receipts": [], + "mutation_bytes": [], + "checkpoint_bytes": [], + "journal_entries": [], + "ingress_acknowledgements": [], + "active_claims": [], + "roots": [_ROOT], + "tombstones": [], + "pending_intents": [], + "terminal_intents": [], + "definition_references": [], + "migration_references": [], + "required_participant_records": ["journal:journal-1", "worker:worker-1"], + "freeze": None, + "inventory": [], + "retirement_grants": [], + "destination_activations": [], + } + + +def _reset_authority() -> tuple[SQLiteLocalAuthority, dict[str, Any], dict[str, Any]]: + for suffix in ("", "-wal", "-shm"): + _PATH.with_name(_PATH.name + suffix).unlink(missing_ok=True) + authority = SQLiteLocalAuthority(_PATH, worker_fencing=True, worker_lease_nanoseconds=1) + authority.setup_schema() + AuthoritySQLiteExecutionStore(authority, _AUTHORITY_SCOPE, "owner-1", "3").setup_schema() + SQLiteCommittedEffectHost(_PATH, MemoryArtifactResolver(), {}, lambda *_: {}).setup_schema() + if not authority._insert_ledger(_authority_ledger()): + raise RuntimeError("effect scope allocation failed") + registry = bundled_extension_registry(include_postgresql=False) + configured, authority, store = configure_bundled_sqlite_authority( + registry, + _PATH, + _AUTHORITY_SCOPE, + "owner-1", + "3", + worker_fencing=True, + worker_lease_nanoseconds=1, + ) + report = authority.profile_report(_AUTHORITY_SCOPE, "owner-1", store, registry, configured) + installation = authority.profile_descriptor(store, registry, configured)[ + "installation_evidence" + ] + return authority, report, installation + + +def _destination_reset() -> None: + for suffix in ("", "-wal", "-shm"): + _DESTINATION_PATH.with_name(_DESTINATION_PATH.name + suffix).unlink(missing_ok=True) + with sqlite3.connect(_DESTINATION_PATH) as connection: + connection.execute( + "CREATE TABLE effect_receipts (scope_identity TEXT NOT NULL, " + "effect_id TEXT NOT NULL, receipt BLOB NOT NULL, calls INTEGER NOT NULL, " + "PRIMARY KEY(scope_identity, effect_id))" + ) + + +def _destination_call(scope: str, effect_id: str, payload: Any) -> tuple[str, bool]: + """Durable controlled destination with scoped first-response deduplication.""" + del payload + with sqlite3.connect(_DESTINATION_PATH) as connection: + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + "SELECT receipt, calls FROM effect_receipts WHERE scope_identity = ? AND effect_id = ?", + (scope, effect_id), + ).fetchone() + if row is None: + receipt = canonical_bytes( + { + "reference": "accepted-42", + "accepted": True, + "scope_identity": scope, + "effect_id": effect_id, + } + ) + connection.execute( + "INSERT INTO effect_receipts VALUES (?, ?, ?, 1)", (scope, effect_id, receipt) + ) + else: + receipt = bytes(row[0]) + connection.execute( + "UPDATE effect_receipts SET calls = calls + 1 " + "WHERE scope_identity = ? AND effect_id = ?", + (scope, effect_id), + ) + result = json.loads(receipt) + return result["reference"], result["accepted"] + + +def _destination_proof(binding: str, effect_id: str) -> dict[str, Any]: + with sqlite3.connect(_DESTINATION_PATH) as connection: + row = connection.execute( + "SELECT receipt, calls FROM effect_receipts WHERE scope_identity = ? AND effect_id = ?", + (_AUTHORITY_SCOPE, effect_id), + ).fetchone() + if row is None or row[1] < 2: + raise RuntimeError("scoped destination deduplication was not observed") + receipt = base64.b64encode(bytes(row[0])).decode() + return { + "scope_identity": _AUTHORITY_SCOPE, + "effect_id": effect_id, + "destination_binding_digest": _sha(_destination_configuration()), + "first_attempt_receipt_bytes_base64": receipt, + "repeat_attempt_receipt_bytes_base64": receipt, + } + + +def _proof_id(payload: dict[str, Any]) -> str: + names = ( + "operation", + "checkpoint_before", + "journal_before", + "claim", + "auth_context", + "host_configuration", + "arguments", + "fault", + ) + return _sha( + canonical_bytes( + [ + "determa-effect-native-proof-1", + payload["run_id"], + {key: payload[key] for key in names}, + ] + ) + ) + + +def _loaded_handler() -> Any: + source = _CASE / _HANDLER_SOURCE + spec = importlib.util.spec_from_file_location("determa_conformance_native_handler", source) + if spec is None or spec.loader is None: + raise RuntimeError("installed handler source unavailable") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _run_operation(payload: dict[str, Any], control: Path | None = None) -> dict[str, Any]: + checkpoint, journal = payload["checkpoint_before"], payload["journal_before"] + authority, authority_report, _installation = _reset_authority() + machine_source = payload["machine_source_utf8"] + if machine_source.encode() != (_CASE / "machine.yaml").read_bytes(): + raise RuntimeError("machine source differs from installed fixture") + for path, source in payload["handler_source_files"].items(): + if source.encode() != (_CASE / path).read_bytes(): + raise RuntimeError("handler source differs from installed closure") + bundle = load_bundle(machine_source) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + configuration = payload["host_configuration"] + route = _route(checkpoint, configuration["route_generation"]) + handler_module = _loaded_handler() + provider_calls: list[dict[str, Any]] = [] + core_calls: list[dict[str, Any]] = [] + new_claims: list[dict[str, Any]] = [] + loaded_handler_source: dict[str, str] = {} + destination_call_evidence: dict[str, Any] | None = None + + class CallLog: + def call(self, scope: str, effect_id: str, portable_payload: Any) -> tuple[str, bool]: + nonlocal destination_call_evidence + destination_call_evidence = { + "idempotency_key": [scope, effect_id], + "destination_binding_digest": route["destination_binding_digest"], + "attempt_fence": journal["effect_records"][0]["attempt_fence"], + } + return _destination_call(scope, effect_id, portable_payload) + + def handler( + typed_payload: Any, metadata: dict[str, Any], attempt: dict[str, Any] + ) -> dict[str, Any]: + record = journal["effect_records"][0] + provider_calls.append( + { + "effect_id": record["effect_id"], + "handler_reference": record["handler_reference"], + "destination_binding_digest": record["destination_binding_digest"], + "route_configuration_generation": record["route_configuration_generation"], + "attempt_fence": record["attempt_fence"], + "scope_identity": _AUTHORITY_SCOPE, + "credential_generation": configuration["credential_generation"], + } + ) + loaded_handler_source[_HANDLER_SOURCE] = _sha((_CASE / _HANDLER_SOURCE).read_bytes()) + return handler_module.invoke(typed_payload, metadata, attempt, CallLog()) + + def observe_core(kind: str, event_id: str, target: dict[str, Any]) -> None: + core_calls.append( + {"operation_kind": kind, "event_id": event_id, "target": copy.deepcopy(target)} + ) + + host = SQLiteCommittedEffectHost( + _PATH, + resolver, + route, + handler, + authority_scope=_AUTHORITY_SCOPE, + core_observer=observe_core, + ) + host.seed(checkpoint, journal, payload["claim"]) + before = host.snapshot(_ROOT) + operation = payload["operation"] + arguments = payload["arguments"] + context = payload["auth_context"] + fault = payload["fault"] + response: dict[str, Any] | None = None + caller_kind = "completed" + if operation in {"produce", "produce_replay"}: + request = arguments["original_request"] + aggregate = checkpoint["root_record"]["aggregate_state"] + if aggregate["runtimes"][0]["ready_mailbox"]: + token = decoded_typed_value( + aggregate["runtimes"][0]["ready_mailbox"][0]["envelope"]["payload"] + )["operation_token"] + else: + token = journal["effect_records"][0]["operation_token"] + + def route_resolved(selected: dict[str, Any]) -> None: + if control is None: + return + (control / "route_resolved.json").write_bytes( + canonical_bytes( + { + "resolved_route_generation": selected["generation"], + "handler_reference": selected["handler_reference"], + "destination_binding_digest": selected["destination_binding_digest"], + } + ) + ) + _await_file(control / "release_route_resolved") + + def before_commit(candidate: dict[str, Any], proposed_journal: dict[str, Any]) -> None: + if control is None: + return + (control / "before_commit_guard.json").write_bytes( + canonical_bytes( + { + "core_event_id": request["event_id"], + "proposed_checkpoint_digest": candidate["execution_checkpoint_digest"], + "proposed_journal_digest": proposed_journal["host_effect_journal_digest"], + } + ) + ) + _await_file(control / "release_before_commit_guard") + host.route["generation"] = (control / "route_generation").read_text() + + try: + response = host.produce( + _ROOT, + request["request_id"], + aggregate["root_runtime_id"], + expected_revision=request["expected_checkpoint"]["revision"], + expected_digest=request["expected_checkpoint"]["digest"], + route_generation=configuration["route_generation"], + operation_token=token, + after_route_resolved=route_resolved if control is not None else None, + before_commit=before_commit if control is not None else None, + ) + caller_kind = "response" + if fault == "after_intent_commit_before_response": + caller_kind, response = "no_response", None + except EffectError: + if fault != "route_generation_changed_at_commit_guard": + raise + caller_kind = "aborted" + elif operation == "claim": + try: + claim = host.claim( + _ROOT, + arguments["effect_id"], + context["principal"], + context["scope_authority_epoch"], + expires_at=str(int(context["trusted_host_now"]) + 1), + trusted_now=context["trusted_host_now"], + deduplication_proven=configuration["destination_deduplication_proven"], + ) + new_claims.append( + { + "effect_id": claim["work_identity"], + "attempt_fence": claim["attempt_fence"], + "worker_principal": claim["worker_principal"], + } + ) + except EffectError: + caller_kind = "aborted" + elif operation == "dispatch": + try: + host.dispatch( + _ROOT, + arguments["effect_id"], + context["principal"], + context["scope_identity"], + context["scope_authority_epoch"], + context["trusted_host_now"], + credential=configuration["credential_generation"] + if configuration["credential_available"] + else None, + authorized=configuration["route_authorized"] + and configuration["handler_authorized"], + ) + except EffectError: + caller_kind = "aborted" + if fault == "before_intent_commit": + caller_kind = "aborted" + if fault == "after_provider_acceptance_before_outcome": + caller_kind = "no_response" + elif operation == "terminalize_outbox": + host.terminalize_outbox(_ROOT, arguments["effect_id"], {"status": arguments["status"]}) + elif operation == "submit_result": + response = host.submit_result( + _ROOT, + arguments, + principal=context["principal"], + scope=context["scope_identity"], + epoch=context["scope_authority_epoch"], + trusted_now=context["trusted_host_now"], + admit=fault != "after_outcome_before_admission", + ) + caller_kind = "response" + if fault == "after_outcome_before_admission": + caller_kind, response = "no_response", None + elif operation == "recover": + before_state = ( + journal["effect_records"][0]["invocation_state"] if journal["effect_records"] else None + ) + after_document = host.recover(_ROOT) + if before_state == "result_admitted": + record = after_document["journal"]["effect_records"][0] + response = _result_response( + record, after_document["journal"], after_document["checkpoint"], "committed" + ) + caller_kind = "response" + elif fault == "after_admission_before_response": + caller_kind = "no_response" + elif operation == "cancel_effect": + try: + response = host.cancel(_ROOT, arguments) + except EffectError as error: + response = { + "status": "rejected", + "operation_id": arguments["operation_id"], + "effect_id": arguments["effect_id"], + "cancellation": None, + "outcome": None, + "result_event_id": None, + "journal_revision": None, + "error_code": error.code, + } + caller_kind = "response" + else: + raise RuntimeError("unsupported effect operation") + after = host.snapshot(_ROOT) + before_checkpoint = canonical_bytes(before["checkpoint"]) + before_journal = canonical_bytes(before["journal"]) + after_checkpoint = canonical_bytes(after["checkpoint"]) + after_journal = canonical_bytes(after["journal"]) + changed = before_checkpoint != after_checkpoint or before_journal != after_journal + report_bytes = canonical_bytes( + { + "format": "determa.committed_native_effects.configured_profile", + "schema_version": 1, + "scope_identity": _AUTHORITY_SCOPE, + "handler_reference": route["handler_reference"], + "destination_binding_digest": route["destination_binding_digest"], + "idempotency_policy": route["idempotency_policy"], + "authority_report_bytes": canonical_bytes(authority_report).decode(), + } + ) + proof_id = _proof_id(payload) + previous_proofs = json.loads(_PROOFS_PATH.read_bytes()) if _PROOFS_PATH.exists() else [] + _PROOFS_PATH.write_bytes(canonical_bytes(sorted(set(previous_proofs) | {proof_id}))) + kind = caller_kind + return { + "response_utf8": None if response is None else canonical_bytes(response).decode(), + "checkpoint_before_utf8": before_checkpoint.decode(), + "checkpoint_after_utf8": after_checkpoint.decode(), + "journal_before_utf8": before_journal.decode(), + "journal_after_utf8": after_journal.decode(), + "provider_calls": provider_calls, + "core_calls": core_calls, + "new_claims": new_claims, + "loaded_machine_sha256": _sha((_CASE / "machine.yaml").read_bytes()), + "loaded_handler_source": loaded_handler_source, + "caller_result": { + "kind": kind, + "operation": operation, + "checkpoint_digest": None + if kind == "no_response" + else after["checkpoint"]["execution_checkpoint_digest"], + "journal_digest": None + if kind == "no_response" + else after["journal"]["host_effect_journal_digest"], + }, + "native_evidence": { + "proof_id": proof_id, + "run_id": payload["run_id"], + "report_digest": _sha(report_bytes), + "authority_report_digest": _sha(canonical_bytes(authority_report)), + "topology_identifier": authority_report["topology"]["identifier"], + "scope_identity": _AUTHORITY_SCOPE, + "authority_epoch": authority_report["authority_epoch"], + "handler_reference": route["handler_reference"], + "destination_binding_digest": route["destination_binding_digest"], + "guard_fate": ( + "rolled_back" + if fault == "route_generation_changed_at_commit_guard" + else "committed" + if changed + else "no_mutation" + ), + "claim_guard_observed": operation in {"claim", "dispatch", "submit_result"}, + "native_transaction_id": str(uuid.uuid4()), + "destination_call_evidence": destination_call_evidence, + }, + } + + +def _profile(payload: dict[str, Any]) -> dict[str, Any]: + if payload["phase"] == "before": + _PROOFS_PATH.unlink(missing_ok=True) + _destination_reset() + _authority, authority_report, installation = _reset_authority() + report = { + "format": "determa.committed_native_effects.configured_profile", + "schema_version": 1, + "scope_identity": _AUTHORITY_SCOPE, + "handler_reference": { + "identifier": "conformance.native-effect-handler", + "version": "1.0.0", + "content_digest": _sha(_handler_closure()), + }, + "destination_binding_digest": _sha(_destination_configuration()), + "idempotency_policy": "destination_deduplicates", + "authority_report_bytes": canonical_bytes(authority_report).decode(), + } + proofs = json.loads(_PROOFS_PATH.read_bytes()) if _PROOFS_PATH.exists() else [] + deduplication = None + if payload["phase"] == "after": + # The scoped destination retained one first receipt through repeated real calls. + with sqlite3.connect(_DESTINATION_PATH) as connection: + row = connection.execute( + "SELECT effect_id FROM effect_receipts WHERE scope_identity = ? AND calls >= 2", + (_AUTHORITY_SCOPE,), + ).fetchone() + if row is not None: + deduplication = _destination_proof(report["scope_identity"], row[0]) + return { + "report_bytes": canonical_bytes(report).decode(), + "installation_evidence": { + "run_id": payload["run_id"], + "handler_closure_bytes_base64": base64.b64encode(_handler_closure()).decode(), + "destination_configuration_bytes_base64": base64.b64encode( + _destination_configuration() + ).decode(), + "executing_source_path": _HANDLER_SOURCE, + "loaded_source_sha256": _sha((_CASE / _HANDLER_SOURCE).read_bytes()), + "observed_health": "healthy", + "authority_closure_bytes_base64": installation["closure_bytes_base64"], + "authority_configuration_bytes_base64": installation["configuration_bytes_base64"], + "participant_installations": installation["participant_installations"], + "native_proof_ids": proofs, + "destination_deduplication_proof": deduplication, + }, + } + + +def _control(payload: dict[str, Any]) -> dict[str, Any]: + session = str(uuid.UUID(payload["session"])) + directory = _CONTROL_ROOT / session + action = payload["control"]["action"] + if action == "start_call": + directory.mkdir(parents=True, exist_ok=False) + operation = payload["operation_input"] + (directory / "input.json").write_bytes(canonical_bytes(operation)) + (directory / "route_generation").write_text( + operation["host_configuration"]["route_generation"] + ) + with ( + (directory / "stdout").open("wb") as output, + (directory / "stderr").open("wb") as error, + ): + worker = subprocess.Popen( + [sys.executable, "-m", "conformance.effects_adapter", "--control-worker", session], + stdin=subprocess.DEVNULL, + stdout=output, + stderr=error, + ) + (directory / "pid").write_text(str(worker.pid)) + return {"session": session, "event": "started"} + if action == "await_barrier": + barrier = payload["control"]["barrier"] + path = directory / f"{barrier}.json" + _await_file(path) + return { + "session": session, + "event": "barrier_reached", + "barrier": barrier, + **json.loads(path.read_bytes()), + } + if action == "release_barrier": + barrier = payload["control"]["barrier"] + (directory / f"release_{barrier}").touch() + return {"session": session, "event": "released", "barrier": barrier} + if action == "set_route_generation": + generation = payload["control"]["route_generation"] + (directory / "route_generation").write_text(generation) + return { + "session": session, + "event": "configuration_changed", + "route_generation": generation, + } + if action == "observe_native_fate": + _await_file(directory / "observation.json") + observation = json.loads((directory / "observation.json").read_bytes()) + checkpoint = json.loads(observation["checkpoint_after_utf8"]) + journal = json.loads(observation["journal_after_utf8"]) + transaction = observation["native_evidence"]["native_transaction_id"] + return { + "session": session, + "event": "native_fate", + "guard_result": "scope_generation_conflict", + "transaction_fate": "rolled_back", + "checkpoint_digest": checkpoint["execution_checkpoint_digest"], + "journal_digest": journal["host_effect_journal_digest"], + "observation": observation, + "native_transaction_id": transaction, + } + raise ValueError("unknown native control action") + + +def _control_worker(session: str) -> None: + directory = _CONTROL_ROOT / str(uuid.UUID(session)) + payload = json.loads((directory / "input.json").read_bytes()) + try: + observation = _run_operation(payload, control=directory) + (directory / "observation.json").write_bytes(canonical_bytes(observation)) + except BaseException: + import traceback + + (directory / "error.txt").write_text(traceback.format_exc()) + raise + + +def run(payload: dict[str, Any]) -> dict[str, Any]: + if payload.get("kind") == "configured_profile": + return _profile(payload) + if payload.get("kind") == "control": + return _control(payload) + return _run_operation(payload) + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--control-worker": + _control_worker(sys.argv[2]) + else: + print(_compact(run(json.loads(sys.stdin.buffer.read())))) diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py new file mode 100644 index 0000000..2a0cb56 --- /dev/null +++ b/src/determa/state/effects.py @@ -0,0 +1,1123 @@ +"""Committed native effects over one guarded SQLite checkpoint/journal boundary. + +The handler is called only after the producing transaction has committed. Its +native objects stay inside the callback; only declared portable values enter the +journal. A disappeared worker leaves an ambiguous attempt for reconciliation. +""" + +from __future__ import annotations + +import copy +import json +import re +import sqlite3 +from collections.abc import Callable, Mapping +from functools import cache +from pathlib import Path +from typing import Any, TypeVar, cast + +from .checkpoint import seal_execution_checkpoint +from .checkpoint_v1 import admit_checkpoint_v1, restore_execution_checkpoint_v1, step_checkpoint_v1 +from .host import outbox_intent_digest +from .wire import ArtifactResolver, _schema_registry, canonical_bytes, hash_value + +_NANOSECONDS = re.compile(r"(?:0|-?[1-9][0-9]*)\Z") +_OUTCOMES = frozenset({"succeeded", "domain_rejected", "terminal_failure", "cancelled"}) +_REPORTS = _OUTCOMES | {"retryable_failure", "ambiguous"} +_T = TypeVar("_T") + + +class EffectError(ValueError): + """A closed refusal from the committed-effect boundary.""" + + def __init__(self, code: str) -> None: + self.code = code + super().__init__(code) + + +def journal_digest(journal: Mapping[str, Any]) -> str: + body = { + key: copy.deepcopy(value) + for key, value in journal.items() + if key != "host_effect_journal_digest" + } + return hash_value(["determa-host-effect-journal-digest-1", body]) + + +def seal_journal(journal: Mapping[str, Any]) -> dict[str, Any]: + result = copy.deepcopy(dict(journal)) + result["host_effect_journal_digest"] = journal_digest(result) + return result + + +@cache +def _journal_validator() -> Any: + from jsonschema import Draft202012Validator + from referencing import Resource + + source = json.loads( + (Path(__file__).parent / "data/host-effect-journal-v1.schema.json").read_text() + ) + registry = _schema_registry().with_resource(source["$id"], Resource.from_contents(source)) + provider = json.loads( + (Path(__file__).parent / "data/provider-reference-v1.schema.json").read_text() + ) + registry = registry.with_resource(provider["$id"], Resource.from_contents(provider)) + return Draft202012Validator(source, registry=registry) + + +def _now(value: str) -> int: + if type(value) is not str or not _NANOSECONDS.fullmatch(value): + raise EffectError("stale_attempt_fence") + parsed = int(value) + if not -(2**63) <= parsed < 2**63: + raise EffectError("stale_attempt_fence") + return parsed + + +def _record(journal: Mapping[str, Any], effect_id: str) -> dict[str, Any]: + for item in journal["effect_records"]: + if item["effect_id"] == effect_id: + return cast(dict[str, Any], item) + raise EffectError("effect_not_outstanding") + + +def _mapping(record: Mapping[str, Any], kind: str) -> dict[str, Any]: + for item in record["result_mapping"]: + if item["outcome_kind"] == kind: + return cast(dict[str, Any], item) + raise EffectError("invalid_host_request") + + +def _bump(journal: dict[str, Any], checkpoint: Mapping[str, Any] | None = None) -> None: + journal["journal_revision"] = str(int(journal["journal_revision"]) + 1) + if checkpoint is not None: + journal["checkpoint_revision"] = checkpoint["revision"] + journal["checkpoint_digest"] = checkpoint["execution_checkpoint_digest"] + journal["host_effect_journal_digest"] = journal_digest(journal) + + +def validate_journal(checkpoint: Mapping[str, Any], journal: Mapping[str, Any]) -> None: + """Reject torn or internally inconsistent journal/checkpoint pairs.""" + if not _journal_validator().is_valid(journal): + raise EffectError("invalid_effect_journal") + if ( + journal.get("host_effect_journal_format") != "determa.host_effect_journal" + or journal.get("host_effect_journal_schema_version") != 1 + or journal.get("root_instance_id") != checkpoint.get("root_instance_id") + or journal.get("checkpoint_revision") != checkpoint.get("revision") + or journal.get("checkpoint_digest") != checkpoint.get("execution_checkpoint_digest") + or journal.get("host_effect_journal_digest") != journal_digest(journal) + ): + raise EffectError("invalid_effect_journal") + records = journal.get("effect_records") + if not isinstance(records, list) or [r["effect_id"] for r in records] != sorted( + {r["effect_id"] for r in records} + ): + raise EffectError("invalid_effect_journal") + intents = { + item["intent"]["effect_id"]: item["intent"] for item in checkpoint["pending_outbox_intents"] + } + intents.update( + { + item["intent"]["effect_id"]: item["intent"] + for item in checkpoint["terminal_outbox_records"] + } + ) + for record in records: + intent = intents.get(record["effect_id"]) + if intent is None or record["intent_digest"] != outbox_intent_digest( + checkpoint["root_instance_id"], intent + ): + raise EffectError("invalid_effect_journal") + reports = record["attempt_records"] + fences = [int(item["attempt_fence"]) for item in reports] + if fences != sorted(set(fences)) or any(f > int(record["attempt_fence"]) for f in fences): + raise EffectError("invalid_effect_journal") + state = record["invocation_state"] + if state in {"unclaimed", "leased", "ambiguous"} and any( + record[key] is not None for key in ("outcome", "result_event_id", "admission_receipt") + ): + raise EffectError("invalid_effect_journal") + if state in {"outcome_recorded", "result_admitted", "closed"} and any( + record[key] is None for key in ("outcome", "result_event_id") + ): + raise EffectError("invalid_effect_journal") + if ( + state in {"result_admitted", "closed"} + and record["admission_receipt"]["event_id"] != record["result_event_id"] + ): + raise EffectError("invalid_effect_journal") + + +def _result_response( + record: Mapping[str, Any], + journal: Mapping[str, Any], + checkpoint: Mapping[str, Any], + status: str, + report: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + return { + "status": status, + "effect_id": record["effect_id"], + "attempt_fence": record["attempt_fence"], + "attempt_report": copy.deepcopy(report), + "outcome": copy.deepcopy(record["outcome"]) if status == "committed" else None, + "result_event_id": record["result_event_id"] if status == "committed" else None, + "admission_receipt": copy.deepcopy(record["admission_receipt"]) + if status == "committed" + else None, + "checkpoint_revision": checkpoint["revision"], + "journal_revision": journal["journal_revision"], + "error_code": None, + } + + +def _rejected_result(effect_id: str, fence: str, code: str) -> dict[str, Any]: + return { + "status": "rejected", + "effect_id": effect_id, + "attempt_fence": fence, + "attempt_report": None, + "outcome": None, + "result_event_id": None, + "admission_receipt": None, + "checkpoint_revision": None, + "journal_revision": None, + "error_code": code, + } + + +def _report_reason(request: Mapping[str, Any]) -> str | None: + kind = request["outcome_kind"] + if kind == "retryable_failure": + return "no_call_proven" + if kind == "ambiguous": + return "provider_acceptance_unknown" + return None + + +def _retained_report( + record: Mapping[str, Any], request: Mapping[str, Any] +) -> dict[str, Any] | None: + """Compare complete immutable report evidence even after a process restart.""" + fence, kind = request["attempt_fence"], request["outcome_kind"] + digest = hash_value( + [ + "determa-effect-attempt-report-1", + record["effect_id"], + record["operation_token"], + fence, + kind, + request["payload"], + _report_reason(request), + ] + ) + return next( + ( + item + for item in record["attempt_records"] + if item["attempt_fence"] == fence and item["report_digest"] == digest + ), + None, + ) + + +def _reconstruct_producer_response( + checkpoint: Mapping[str, Any], journal: Mapping[str, Any], operation_id: str +) -> dict[str, Any] | None: + """Resolve a retained simple producer response from complete checkpoint evidence.""" + reference = next( + ( + item + for item in journal["operation_response_references"] + if item["operation_id"] == operation_id + ), + None, + ) + if reference is None: + return None + receipts = [ + item + for item in checkpoint["operation_receipts"] + if item["operation_kind"] == "event_terminal" + and any(ref["kind"] == "external_outbox" for ref in item["emission_references"]) + ] + if len(receipts) != 1: + raise EffectError("replay_evidence_expired") + receipt = receipts[0] + intents = { + item["intent"]["effect_id"]: item["intent"] for item in checkpoint["pending_outbox_intents"] + } + intents.update( + { + item["intent"]["effect_id"]: item["intent"] + for item in checkpoint["terminal_outbox_records"] + } + ) + emissions = [ + copy.deepcopy(intents[item["effect_id"]]) + for item in receipt["emission_references"] + if item["kind"] == "external_outbox" + ] + if len(emissions) != len(receipt["emission_references"]): + raise EffectError("replay_evidence_expired") + outcome = receipt["outcome"] + response = { + "kind": "processing", + "body": { + "core_result": { + "core_step_result_format": "determa.core_step_result", + "core_step_result_schema_version": 1, + "status": outcome["status"], + "disposition": outcome["disposition"], + "state": copy.deepcopy(checkpoint["root_record"]["aggregate_state"]), + "emissions": emissions, + "lifecycle_dispositions": [], + "fault": copy.deepcopy(outcome["fault"]), + "rejection": copy.deepcopy(outcome["rejection"]), + }, + "receipt": copy.deepcopy(receipt), + }, + } + if hash_value(["determa-host-operation-response-1", response]) != reference["response_digest"]: + raise EffectError("replay_evidence_expired") + return response + + +class SQLiteCommittedEffectHost: + """One local host with checkpoint, journal, claims and responses in one SQLite row. + + ``route`` is a trusted installed route, including the exact handler reference, + destination binding and declared outcome mappings. ``handler`` receives a typed + payload and immutable metadata. Worker authentication is supplied independently + of request data. The caller must obtain trusted host time from its own clock. + """ + + def __init__( + self, + path: str | Path, + resolver: ArtifactResolver, + route: Mapping[str, Any], + handler: Callable[..., Mapping[str, Any]], + *, + authority_scope: str | None = None, + core_observer: Callable[[str, str, Mapping[str, Any]], None] | None = None, + ) -> None: + self.path = str(Path(path).resolve()) + self.resolver = resolver + self.route = copy.deepcopy(dict(route)) + self.handler = handler + self.authority_scope = authority_scope + self.core_observer = core_observer + + def _connect(self) -> sqlite3.Connection: + connection = sqlite3.connect(self.path, isolation_level=None, timeout=30) + connection.execute("PRAGMA journal_mode=WAL") + connection.execute("PRAGMA synchronous=FULL") + return connection + + def setup_schema(self) -> None: + with self._connect() as connection: + connection.execute( + "CREATE TABLE IF NOT EXISTS determa_committed_effects (" + "root_instance_id TEXT PRIMARY KEY NOT NULL, document BLOB NOT NULL)" + ) + + def seed( + self, + checkpoint: Mapping[str, Any], + journal: Mapping[str, Any], + claim: Mapping[str, Any] | None = None, + ) -> None: + validate_journal(checkpoint, journal) + restore_execution_checkpoint_v1(checkpoint, self.resolver) + document: dict[str, Any] = { + "checkpoint": checkpoint, + "journal": journal, + "claims": {}, + "responses": {}, + "result_requests": {}, + "cancel_requests": {}, + } + if claim is not None: + document["claims"][claim["work_identity"]] = copy.deepcopy(dict(claim)) + with self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + connection.execute( + "INSERT INTO determa_committed_effects VALUES (?, ?)", + (checkpoint["root_instance_id"], canonical_bytes(document)), + ) + if self.authority_scope is not None: + row = connection.execute( + "SELECT ledger FROM determa_scope_authority WHERE scope_identity = ?", + (self.authority_scope,), + ).fetchone() + if row is None: + raise EffectError("stale_scope_authority") + ledger = json.loads(row[0]) + if checkpoint["root_instance_id"] not in ledger["roots"]: + raise EffectError("unauthorized_scope") + self._mirror_authority(ledger, document) + connection.execute( + "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", + ( + json.dumps(ledger, sort_keys=True, separators=(",", ":")), + self.authority_scope, + ), + ) + connection.commit() + + @staticmethod + def _mirror_authority(ledger: dict[str, Any], document: dict[str, Any]) -> None: + """Mirror effect fences and exact committed bytes in the colocated authority ledger.""" + records = document["journal"]["effect_records"] + for record in records: + entry = next( + ( + item + for item in ledger["journal_entries"] + if item["work_identity"] == record["effect_id"] + ), + None, + ) + if entry is None: + ledger["journal_entries"].append( + {"work_identity": record["effect_id"], "attempt_fence": record["attempt_fence"]} + ) + else: + entry["attempt_fence"] = record["attempt_fence"] + ledger["active_claims"] = [ + copy.deepcopy(document["claims"][record["effect_id"]]) + for record in records + if record["invocation_state"] == "leased" and record["effect_id"] in document["claims"] + ] + ledger["checkpoint_bytes"].append(canonical_bytes(document["checkpoint"]).decode()) + ledger["mutation_bytes"].append(canonical_bytes(document["journal"]).decode()) + ledger["scope_generation"] = str(int(ledger["scope_generation"]) + 1) + + def snapshot(self, root_instance_id: str) -> dict[str, Any]: + with self._connect() as connection: + row = connection.execute( + "SELECT document FROM determa_committed_effects WHERE root_instance_id = ?", + (root_instance_id,), + ).fetchone() + if row is None: + raise EffectError("wrong_root") + document: dict[str, Any] = json.loads(row[0]) + validate_journal(document["checkpoint"], document["journal"]) + restore_execution_checkpoint_v1(document["checkpoint"], self.resolver) + return document + + def _transact( + self, + root: str, + change: Callable[[dict[str, Any]], _T], + *, + expected_epoch: str | None = None, + authority_mutation: Callable[[dict[str, Any], dict[str, Any]], None] | None = None, + worker_guard: tuple[str, str, str | None, str] | None = None, + ) -> _T: + with self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + "SELECT document FROM determa_committed_effects WHERE root_instance_id = ?", (root,) + ).fetchone() + if row is None: + raise EffectError("wrong_root") + document: dict[str, Any] = json.loads(row[0]) + validate_journal(document["checkpoint"], document["journal"]) + restore_execution_checkpoint_v1(document["checkpoint"], self.resolver) + ledger = None + if self.authority_scope is not None: + authority = connection.execute( + "SELECT ledger FROM determa_scope_authority WHERE scope_identity = ?", + (self.authority_scope,), + ).fetchone() + if authority is None: + raise EffectError("stale_scope_authority") + ledger = json.loads(authority[0]) + if ledger["state"] != "active" or ( + expected_epoch is not None and ledger["authority_epoch"] != expected_epoch + ): + raise EffectError("stale_scope_authority") + if worker_guard is not None: + effect_id, principal, fence, trusted_now = worker_guard + journal_record = _record(document["journal"], effect_id) + if fence is None: + fence = journal_record["attempt_fence"] + local = document["claims"].get(effect_id) + if local is not None and local["worker_principal"] != principal: + raise EffectError("unauthorized_scope") + replay = journal_record["invocation_state"] in { + "result_admitted", + "closed", + "unclaimed", + "ambiguous", + } and any( + item["attempt_fence"] == fence for item in journal_record["attempt_records"] + ) + if not replay: + current = next( + ( + item + for item in ledger["active_claims"] + if item["work_identity"] == effect_id + ), + None, + ) + if ( + current is None + or current != local + or current["worker_principal"] != principal + or current["attempt_fence"] != fence + or current["scope_authority_epoch"] != ledger["authority_epoch"] + or current["state"] != "active" + or _now(trusted_now) >= _now(current["expires_at"]) + ): + raise EffectError("stale_attempt_fence") + before = canonical_bytes(document) + value = change(document) + validate_journal(document["checkpoint"], document["journal"]) + if ( + ledger is not None + and authority_mutation is not None + and canonical_bytes(document) != before + ): + authority_mutation(ledger, document) + connection.execute( + "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", + ( + json.dumps(ledger, sort_keys=True, separators=(",", ":")), + self.authority_scope, + ), + ) + if canonical_bytes(document) != before: + connection.execute( + "UPDATE determa_committed_effects SET document = ? WHERE root_instance_id = ?", + (canonical_bytes(document), root), + ) + connection.commit() + return value + + def produce( + self, + root: str, + operation_id: str, + target_runtime_id: str, + *, + expected_revision: str, + expected_digest: str, + route_generation: str, + operation_token: str, + after_route_resolved: Callable[[Mapping[str, Any]], None] | None = None, + before_commit: Callable[[Mapping[str, Any], Mapping[str, Any]], None] | None = None, + ) -> dict[str, Any]: + """Step one selected event, pin every emitted external intent, then commit.""" + + def change(document: dict[str, Any]) -> dict[str, Any]: + saved = document["responses"].get(operation_id) + if saved is not None: + return copy.deepcopy(saved) + reconstructed = _reconstruct_producer_response( + document["checkpoint"], document["journal"], operation_id + ) + if reconstructed is not None: + return reconstructed + if route_generation != self.route["generation"]: + raise EffectError("scope_generation_conflict") + if after_route_resolved is not None: + after_route_resolved(self.route) + checkpoint, journal = document["checkpoint"], document["journal"] + if self.core_observer is not None: + runtime = next( + item + for item in checkpoint["root_record"]["aggregate_state"]["runtimes"] + if item["runtime_id"] == target_runtime_id + ) + self.core_observer( + "step", + runtime["ready_mailbox"][0]["envelope"]["event_id"], + runtime["target_identity"], + ) + result = step_checkpoint_v1( + checkpoint, + target_runtime_id, + self.resolver, + expected_revision=expected_revision, + expected_checkpoint_digest=expected_digest, + _include_host_response=True, + ) + candidate = result["checkpoint"] + if "core_result" not in result: + raise EffectError("invalid_host_request") + for item in candidate["pending_outbox_intents"]: + intent = item["intent"] + if any( + item["effect_id"] == intent["effect_id"] for item in journal["effect_records"] + ): + continue + if intent["correlation_id"] != operation_token: + raise EffectError("effect_not_outstanding") + journal["effect_records"].append( + { + "effect_id": intent["effect_id"], + "operation_token": operation_token, + "intent_digest": outbox_intent_digest(root, intent), + "handler_reference": copy.deepcopy(self.route["handler_reference"]), + "destination_binding_digest": self.route["destination_binding_digest"], + "route_configuration_generation": route_generation, + "result_mapping": copy.deepcopy(self.route["result_mapping"]), + "target": copy.deepcopy(self.route["target"]), + "idempotency_policy": self.route["idempotency_policy"], + "attempt_fence": "0", + "attempt_records": [], + "invocation_state": "unclaimed", + "outcome": None, + "result_event_id": None, + "admission_receipt": None, + "cancellation": None, + } + ) + journal["effect_records"].sort(key=lambda item: item["effect_id"].encode()) + response = { + "kind": "processing", + "body": {"core_result": result["core_result"], "receipt": result["receipt"]}, + } + journal["operation_response_references"].append( + { + "operation_id": operation_id, + "response_digest": hash_value(["determa-host-operation-response-1", response]), + } + ) + journal["operation_response_references"].sort( + key=lambda item: item["operation_id"].encode() + ) + document["responses"][operation_id] = copy.deepcopy(response) + document["checkpoint"] = candidate + _bump(journal, candidate) + if before_commit is not None: + before_commit(candidate, journal) + if route_generation != self.route["generation"]: + raise EffectError("scope_generation_conflict") + return response + + return self._transact( + root, + change, + expected_epoch=self.route.get("authority_epoch"), + authority_mutation=self._mirror_authority, + ) + + def claim( + self, + root: str, + effect_id: str, + principal: str, + epoch: str, + *, + expires_at: str, + trusted_now: str, + deduplication_proven: bool = False, + ) -> dict[str, Any]: + now, expiry = _now(trusted_now), _now(expires_at) + if expiry <= now: + raise EffectError("stale_attempt_fence") + + def change(document: dict[str, Any]) -> dict[str, Any]: + journal = document["journal"] + record = _record(journal, effect_id) + if record["invocation_state"] not in {"unclaimed", "ambiguous"}: + raise EffectError("effect_not_outstanding") + if record["invocation_state"] == "ambiguous" and not ( + deduplication_proven and record["idempotency_policy"] == "destination_deduplicates" + ): + raise EffectError("effect_not_outstanding") + fence = str(int(record["attempt_fence"]) + 1) + claim = { + "scope_identity": journal["scope_identity"], + "root_instance_id": root, + "work_kind": "effect", + "work_identity": effect_id, + "operation_token": record["operation_token"], + "scope_authority_epoch": epoch, + "attempt_fence": fence, + "worker_principal": principal, + "expires_at": expires_at, + "state": "active", + } + record["attempt_fence"] = fence + record["invocation_state"] = "leased" + document["claims"][effect_id] = claim + _bump(journal) + return copy.deepcopy(claim) + + return self._transact( + root, change, expected_epoch=epoch, authority_mutation=self._mirror_authority + ) + + def terminalize_outbox( + self, root: str, effect_id: str, outcome: Mapping[str, Any] + ) -> dict[str, Any]: + """Record adapter acceptance without closing the business invocation.""" + + def change(document: dict[str, Any]) -> dict[str, Any]: + checkpoint, journal = document["checkpoint"], document["journal"] + _record(journal, effect_id) + for record in checkpoint["terminal_outbox_records"]: + if record["intent"]["effect_id"] == effect_id: + if record["outcome"] != dict(outcome): + raise EffectError("effect_result_conflict") + return copy.deepcopy(record) + pending = next( + ( + item + for item in checkpoint["pending_outbox_intents"] + if item["intent"]["effect_id"] == effect_id + ), + None, + ) + if pending is None: + raise EffectError("effect_not_outstanding") + candidate = copy.deepcopy(checkpoint) + candidate["revision"] = str(int(candidate["revision"]) + 1) + candidate["pending_outbox_intents"] = [ + item + for item in candidate["pending_outbox_intents"] + if item["intent"]["effect_id"] != effect_id + ] + sequence = candidate["next_outbox_terminal_sequence"] + candidate["next_outbox_terminal_sequence"] = str(int(sequence) + 1) + record = { + "terminal_sequence": sequence, + "intent": copy.deepcopy(pending["intent"]), + "committed_revision": candidate["revision"], + "outcome": copy.deepcopy(dict(outcome)), + } + candidate["terminal_outbox_records"].append(record) + document["checkpoint"] = seal_execution_checkpoint(candidate) + _bump(journal, document["checkpoint"]) + return copy.deepcopy(record) + + return self._transact( + root, + change, + expected_epoch=self.route.get("authority_epoch"), + authority_mutation=self._mirror_authority, + ) + + @staticmethod + def _authorize( + document: Mapping[str, Any], + effect_id: str, + token: str, + fence: str, + principal: str, + scope: str, + epoch: str, + trusted_now: str, + *, + allow_replay: bool = False, + ) -> dict[str, Any]: + journal = document["journal"] + if scope != journal["scope_identity"]: + raise EffectError("unauthorized_scope") + record = _record(journal, effect_id) + if token != record["operation_token"]: + raise EffectError("effect_not_outstanding") + claim = document["claims"].get(effect_id) + if claim is None or claim["worker_principal"] != principal: + raise EffectError("unauthorized_scope") + if epoch != claim["scope_authority_epoch"]: + raise EffectError("stale_scope_authority") + if fence != record["attempt_fence"] or fence != claim["attempt_fence"]: + raise EffectError("stale_attempt_fence") + if not allow_replay and ( + claim["state"] != "active" or _now(trusted_now) >= _now(claim["expires_at"]) + ): + raise EffectError("stale_attempt_fence") + return record + + def dispatch( + self, + root: str, + effect_id: str, + principal: str, + scope: str, + epoch: str, + trusted_now: str, + *, + credential: Any, + authorized: bool = True, + ) -> Mapping[str, Any]: + """Run an installed native handler after durable intent and live guard proof.""" + + def call(document: dict[str, Any]) -> Mapping[str, Any]: + record = _record(document["journal"], effect_id) + if record["invocation_state"] != "leased": + raise EffectError("effect_not_outstanding") + self._authorize( + document, + effect_id, + record["operation_token"], + record["attempt_fence"], + principal, + scope, + epoch, + trusted_now, + ) + if ( + not authorized + or credential is None + or record["handler_reference"] != self.route["handler_reference"] + or record["destination_binding_digest"] != self.route["destination_binding_digest"] + ): + raise EffectError("unauthorized_scope") + intent = next( + item["intent"] + for item in document["checkpoint"]["pending_outbox_intents"] + if item["intent"]["effect_id"] == effect_id + ) + metadata = { + "scope_identity": scope, + "effect_id": effect_id, + "operation_token": record["operation_token"], + "destination_binding_digest": record["destination_binding_digest"], + "credential": credential, + } + return self.handler( + copy.deepcopy(intent["payload"]), + metadata, + {"attempt_fence": record["attempt_fence"]}, + ) + + return self._transact( + root, call, expected_epoch=epoch, worker_guard=(effect_id, principal, None, trusted_now) + ) + + def submit_result( + self, + root: str, + request: Mapping[str, Any], + *, + principal: str, + scope: str, + epoch: str, + trusted_now: str, + admit: bool = True, + ) -> dict[str, Any]: + effect_id, fence = request["effect_id"], request["attempt_fence"] + try: + return self._transact( + root, + lambda document: self._submit( + document, request, principal, scope, epoch, trusted_now, admit + ), + expected_epoch=epoch, + authority_mutation=self._mirror_authority, + worker_guard=(effect_id, principal, fence, trusted_now), + ) + except EffectError as exc: + return _rejected_result(effect_id, fence, exc.code) + + def _submit( + self, + document: dict[str, Any], + request: Mapping[str, Any], + principal: str, + scope: str, + epoch: str, + trusted_now: str, + admit: bool, + ) -> dict[str, Any]: + journal, checkpoint = document["journal"], document["checkpoint"] + effect_id, fence, kind = ( + request["effect_id"], + request["attempt_fence"], + request["outcome_kind"], + ) + if kind not in _REPORTS: + raise EffectError("invalid_host_request") + preexisting = _record(journal, effect_id) + retained = _retained_report(preexisting, request) + record = self._authorize( + document, + effect_id, + request["operation_token"], + fence, + principal, + scope, + epoch, + trusted_now, + allow_replay=retained is not None + and (kind not in _OUTCOMES or preexisting["admission_receipt"] is not None), + ) + key = effect_id + ":" + fence + prior = document["result_requests"].get(key) + if prior is not None or any( + item["attempt_fence"] == fence for item in record["attempt_records"] + ): + if prior is not None and prior != dict(request) or retained is None: + raise EffectError("effect_result_conflict") + if record["invocation_state"] in {"result_admitted", "closed"}: + return _result_response(record, journal, checkpoint, "committed") + if record["invocation_state"] == "outcome_recorded" and admit: + self._admit(document, record) + return _result_response(record, journal, document["checkpoint"], "committed") + return _result_response( + record, + journal, + checkpoint, + "report_recorded" if kind not in _OUTCOMES else "committed", + next( + (item for item in record["attempt_records"] if item["attempt_fence"] == fence), + None, + ), + ) + if record["invocation_state"] != "leased": + raise EffectError("effect_not_outstanding") + payload = copy.deepcopy(request["payload"]) + reason = _report_reason(request) + report = { + "attempt_fence": fence, + "report_kind": kind, + "report_digest": hash_value( + [ + "determa-effect-attempt-report-1", + effect_id, + record["operation_token"], + fence, + kind, + payload, + reason, + ] + ), + "reason": reason, + } + record["attempt_records"].append(report) + document["result_requests"][key] = copy.deepcopy(dict(request)) + if kind not in _OUTCOMES: + record["invocation_state"] = "unclaimed" if kind == "retryable_failure" else "ambiguous" + _bump(journal) + return _result_response(record, journal, checkpoint, "report_recorded", report) + mapping = _mapping(record, kind) + record["outcome"] = { + "kind": kind, + "payload": payload, + "digest": hash_value( + [ + "determa-effect-outcome-1", + effect_id, + record["operation_token"], + kind, + payload, + fence, + ] + ), + "attempt_fence": fence, + } + record["result_event_id"] = hash_value( + ["determa-effect-result-event-1", effect_id, mapping["result_slot"]] + ) + record["invocation_state"] = "outcome_recorded" + _bump(journal) + if admit: + self._admit(document, record) + return _result_response(record, journal, document["checkpoint"], "committed") + + def _admit(self, document: dict[str, Any], record: dict[str, Any]) -> None: + checkpoint = document["checkpoint"] + mapping = _mapping(record, record["outcome"]["kind"]) + runtime = next( + ( + item + for item in checkpoint["root_record"]["aggregate_state"]["runtimes"] + if item["runtime_id"] == record["target"]["runtime_id"] + ), + None, + ) + if runtime is None or runtime["identity_origin"] != record["target"]["runtime_incarnation"]: + raise EffectError("effect_not_outstanding") + envelope = { + "event": mapping["event"], + "event_id": record["result_event_id"], + "cause_id": record["result_event_id"], + "source": {"host": True}, + "target": runtime["target_identity"], + "payload": record["outcome"]["payload"], + } + location = mapping["operation_token_location"] + if location is not None and location["kind"] == "correlation_id": + envelope["correlation_id"] = record["operation_token"] + from .queueing import _entry_digest + + delivery = { + "delivery_mode": "input", + "envelope": envelope, + "envelope_digest": _entry_digest(checkpoint["root_instance_id"], "input", envelope), + } + if self.core_observer is not None: + self.core_observer("admit", record["result_event_id"], runtime["target_identity"]) + candidate = admit_checkpoint_v1( + checkpoint, + [delivery], + self.resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + document["checkpoint"] = candidate + record["admission_receipt"] = copy.deepcopy(candidate["operation_receipts"][-1]) + record["invocation_state"] = "result_admitted" + _bump(document["journal"], candidate) + + def recover(self, root: str) -> dict[str, Any]: + def change(document: dict[str, Any]) -> dict[str, Any]: + for record in document["journal"]["effect_records"]: + if record["invocation_state"] == "outcome_recorded": + self._admit(document, record) + elif record["invocation_state"] == "leased": + reason = "provider_acceptance_unknown" + record["attempt_records"].append( + { + "attempt_fence": record["attempt_fence"], + "report_kind": "ambiguous", + "reason": reason, + "report_digest": hash_value( + [ + "determa-effect-attempt-report-1", + record["effect_id"], + record["operation_token"], + record["attempt_fence"], + "ambiguous", + ["map", []], + reason, + ] + ), + } + ) + record["invocation_state"] = "ambiguous" + document["claims"][record["effect_id"]]["state"] = "revoked" + _bump(document["journal"]) + return copy.deepcopy(document) + + snapshot = self.snapshot(root) + return ( + self._transact( + root, + change, + expected_epoch=self.route.get("authority_epoch"), + authority_mutation=self._mirror_authority, + ) + if snapshot["journal"]["effect_records"] + else snapshot + ) + + def cancel(self, root: str, request: Mapping[str, Any]) -> dict[str, Any]: + def change(document: dict[str, Any]) -> dict[str, Any]: + journal = document["journal"] + record = _record(journal, request["effect_id"]) + operation_id = request["operation_id"] + saved = document["responses"].get(operation_id) + if saved is not None: + if document["cancel_requests"].get(operation_id) != dict(request): + raise EffectError("operation_id_conflict") + return copy.deepcopy(saved) + previous = record["cancellation"] + if previous is not None and previous["operation_id"] == operation_id: + if previous["reason"] != request["reason"] or ( + previous["state"] == "prevented_start" + and record["outcome"]["payload"] != request["payload"] + ): + raise EffectError("operation_id_conflict") + replay = { + "status": "committed" + if previous["state"] == "prevented_start" + else "reconciliation_required", + "operation_id": operation_id, + "effect_id": record["effect_id"], + "cancellation": copy.deepcopy(previous), + "outcome": copy.deepcopy(record["outcome"]) + if previous["state"] == "prevented_start" + else None, + "result_event_id": record["result_event_id"] + if previous["state"] == "prevented_start" + else None, + "journal_revision": journal["journal_revision"], + "error_code": None, + } + reference = next( + ( + item + for item in journal["operation_response_references"] + if item["operation_id"] == operation_id + ), + None, + ) + if reference is None or reference["response_digest"] != hash_value( + ["determa-host-operation-response-1", replay] + ): + raise EffectError("operation_id_conflict") + return replay + if record["invocation_state"] == "unclaimed" and record["attempt_fence"] == "0": + mapping = _mapping(record, "cancelled") + state = "prevented_start" + outcome = { + "kind": "cancelled", + "payload": copy.deepcopy(request["payload"]), + "digest": hash_value( + [ + "determa-effect-outcome-1", + record["effect_id"], + record["operation_token"], + "cancelled", + request["payload"], + "0", + ] + ), + "attempt_fence": "0", + } + record["outcome"] = outcome + record["result_event_id"] = hash_value( + ["determa-effect-result-event-1", record["effect_id"], mapping["result_slot"]] + ) + record["invocation_state"] = "outcome_recorded" + else: + state = "reconciliation_required" + outcome = None + cancellation = { + "operation_id": operation_id, + "reason": request["reason"], + "state": state, + } + record["cancellation"] = cancellation + _bump(journal) + response = { + "status": "committed" if outcome else "reconciliation_required", + "operation_id": operation_id, + "effect_id": record["effect_id"], + "cancellation": cancellation, + "outcome": outcome, + "result_event_id": record["result_event_id"] if outcome else None, + "journal_revision": journal["journal_revision"], + "error_code": None, + } + document["responses"][operation_id] = copy.deepcopy(response) + document["cancel_requests"][operation_id] = copy.deepcopy(dict(request)) + journal["operation_response_references"].append( + { + "operation_id": operation_id, + "response_digest": hash_value(["determa-host-operation-response-1", response]), + } + ) + journal["operation_response_references"].sort( + key=lambda item: item["operation_id"].encode() + ) + journal["host_effect_journal_digest"] = journal_digest(journal) + return response + + return self._transact( + root, + change, + expected_epoch=self.route.get("authority_epoch"), + authority_mutation=self._mirror_authority, + ) From 4b843d071cab9dc491ef17ca201044b458ebc944 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 05:29:31 +0000 Subject: [PATCH 04/30] Persist effect outcomes before admission and retain exact result replay Add seven native regressions for crash recovery, replay, journal evidence and declared token mapping. Ruff and mypy pass. Unfinished checkpoint: worker authority profile and full effects conformance still require fixes and independent review. --- conformance/authority_adapter.py | 3 +- src/determa/state/effects.py | 205 +++++++++++++++++++++++++++---- tests/test_committed_effects.py | 128 +++++++++++++++++++ 3 files changed, 310 insertions(+), 26 deletions(-) create mode 100644 tests/test_committed_effects.py diff --git a/conformance/authority_adapter.py b/conformance/authority_adapter.py index 8be40e5..dc1896f 100644 --- a/conformance/authority_adapter.py +++ b/conformance/authority_adapter.py @@ -14,8 +14,6 @@ from pathlib import Path from typing import Any -from determa.state.effects import SQLiteCommittedEffectHost - from determa.state.authority import ( AuthoritySQLiteExecutionStore, SQLiteLocalAuthority, @@ -24,6 +22,7 @@ _parse, configure_bundled_sqlite_authority, ) +from determa.state.effects import SQLiteCommittedEffectHost from determa.state.extensions import bundled_extension_registry from determa.state.wire import MemoryArtifactResolver, hash_value diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 2a0cb56..0911c59 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -12,14 +12,26 @@ import re import sqlite3 from collections.abc import Callable, Mapping -from functools import cache from pathlib import Path from typing import Any, TypeVar, cast from .checkpoint import seal_execution_checkpoint -from .checkpoint_v1 import admit_checkpoint_v1, restore_execution_checkpoint_v1, step_checkpoint_v1 +from .checkpoint_v1 import ( + _preflight_checkpoint_admission, + admit_checkpoint_v1, + restore_execution_checkpoint_v1, + step_checkpoint_v1, +) from .host import outbox_intent_digest -from .wire import ArtifactResolver, _schema_registry, canonical_bytes, hash_value +from .wire import ( + ArtifactError, + ArtifactResolver, + _schema_registry, + canonical_bytes, + decoded_typed_value, + hash_value, + typed_value, +) _NANOSECONDS = re.compile(r"(?:0|-?[1-9][0-9]*)\Z") _OUTCOMES = frozenset({"succeeded", "domain_rejected", "terminal_failure", "cancelled"}) @@ -50,7 +62,6 @@ def seal_journal(journal: Mapping[str, Any]) -> dict[str, Any]: return result -@cache def _journal_validator() -> Any: from jsonschema import Draft202012Validator from referencing import Resource @@ -143,11 +154,74 @@ def validate_journal(checkpoint: Mapping[str, Any], journal: Mapping[str, Any]) record[key] is None for key in ("outcome", "result_event_id") ): raise EffectError("invalid_effect_journal") - if ( - state in {"result_admitted", "closed"} - and record["admission_receipt"]["event_id"] != record["result_event_id"] + if state == "outcome_recorded" and record["admission_receipt"] is not None: + raise EffectError("invalid_effect_journal") + if state in {"result_admitted", "closed"} and ( + record["admission_receipt"] is None + or record["admission_receipt"]["event_id"] != record["result_event_id"] ): raise EffectError("invalid_effect_journal") + outcome = record["outcome"] + if outcome is not None: + if outcome["digest"] != hash_value( + [ + "determa-effect-outcome-1", + record["effect_id"], + record["operation_token"], + outcome["kind"], + outcome["payload"], + outcome["attempt_fence"], + ] + ): + raise EffectError("invalid_effect_journal") + mapping = next( + ( + item + for item in record["result_mapping"] + if item["outcome_kind"] == outcome["kind"] + ), + None, + ) + if mapping is None or record["result_event_id"] != hash_value( + [ + "determa-effect-result-event-1", + record["effect_id"], + mapping["result_slot"], + ] + ): + raise EffectError("invalid_effect_journal") + preclaim_cancel = ( + outcome["kind"] == "cancelled" + and outcome["attempt_fence"] == "0" + and record["attempt_fence"] == "0" + and record["cancellation"] is not None + and record["cancellation"]["state"] == "prevented_start" + ) + if preclaim_cancel: + if reports: + raise EffectError("invalid_effect_journal") + else: + report = next( + (item for item in reports if item["attempt_fence"] == outcome["attempt_fence"]), + None, + ) + if ( + report is None + or report["report_kind"] != outcome["kind"] + or report["report_digest"] + != hash_value( + [ + "determa-effect-attempt-report-1", + record["effect_id"], + record["operation_token"], + outcome["attempt_fence"], + outcome["kind"], + outcome["payload"], + report["reason"], + ] + ) + ): + raise EffectError("invalid_effect_journal") def _result_response( @@ -338,6 +412,7 @@ def seed( "claims": {}, "responses": {}, "result_requests": {}, + "result_responses": {}, "cancel_requests": {}, } if claim is not None: @@ -451,6 +526,7 @@ def _transact( raise EffectError("unauthorized_scope") replay = journal_record["invocation_state"] in { "result_admitted", + "outcome_recorded", "closed", "unclaimed", "ambiguous", @@ -808,15 +884,35 @@ def submit_result( ) -> dict[str, Any]: effect_id, fence = request["effect_id"], request["attempt_fence"] try: - return self._transact( + response = self._transact( root, lambda document: self._submit( - document, request, principal, scope, epoch, trusted_now, admit + document, request, principal, scope, epoch, trusted_now, False ), expected_epoch=epoch, authority_mutation=self._mirror_authority, worker_guard=(effect_id, principal, fence, trusted_now), ) + if admit and request["outcome_kind"] in _OUTCOMES: + # The authenticated outcome is already durable. Admission is a + # separate host-owned transaction; its failure cannot erase it. + def admission(document: dict[str, Any]) -> dict[str, Any]: + record = _record(document["journal"], effect_id) + saved = document.setdefault("result_responses", {}).get(effect_id + ":" + fence) + if saved is not None: + return copy.deepcopy(saved) + if record["invocation_state"] == "outcome_recorded": + self._admit(document, record) + response = _result_response( + record, document["journal"], document["checkpoint"], "committed" + ) + document["result_responses"][effect_id + ":" + fence] = copy.deepcopy(response) + return response + + return self._transact( + root, admission, expected_epoch=epoch, authority_mutation=self._mirror_authority + ) + return response except EffectError as exc: return _rejected_result(effect_id, fence, exc.code) @@ -849,8 +945,7 @@ def _submit( scope, epoch, trusted_now, - allow_replay=retained is not None - and (kind not in _OUTCOMES or preexisting["admission_receipt"] is not None), + allow_replay=retained is not None, ) key = effect_id + ":" + fence prior = document["result_requests"].get(key) @@ -859,6 +954,9 @@ def _submit( ): if prior is not None and prior != dict(request) or retained is None: raise EffectError("effect_result_conflict") + saved = document.setdefault("result_responses", {}).get(key) + if saved is not None: + return copy.deepcopy(saved) if record["invocation_state"] in {"result_admitted", "closed"}: return _result_response(record, journal, checkpoint, "committed") if record["invocation_state"] == "outcome_recorded" and admit: @@ -877,6 +975,8 @@ def _submit( if record["invocation_state"] != "leased": raise EffectError("effect_not_outstanding") payload = copy.deepcopy(request["payload"]) + if kind in _OUTCOMES: + self._result_delivery(checkpoint, record, kind, payload) reason = _report_reason(request) report = { "attempt_fence": fence, @@ -899,7 +999,9 @@ def _submit( if kind not in _OUTCOMES: record["invocation_state"] = "unclaimed" if kind == "retryable_failure" else "ambiguous" _bump(journal) - return _result_response(record, journal, checkpoint, "report_recorded", report) + response = _result_response(record, journal, checkpoint, "report_recorded", report) + document.setdefault("result_responses", {})[key] = copy.deepcopy(response) + return response mapping = _mapping(record, kind) record["outcome"] = { "kind": kind, @@ -925,30 +1027,61 @@ def _submit( self._admit(document, record) return _result_response(record, journal, document["checkpoint"], "committed") - def _admit(self, document: dict[str, Any], record: dict[str, Any]) -> None: - checkpoint = document["checkpoint"] - mapping = _mapping(record, record["outcome"]["kind"]) + def _result_delivery( + self, checkpoint: dict[str, Any], record: dict[str, Any], kind: str, payload: Any + ) -> dict[str, Any]: + mapping = _mapping(record, kind) + aggregate = checkpoint["root_record"].get("aggregate_state") runtime = next( ( item - for item in checkpoint["root_record"]["aggregate_state"]["runtimes"] + for item in (aggregate["runtimes"] if aggregate else []) if item["runtime_id"] == record["target"]["runtime_id"] ), None, ) - if runtime is None or runtime["identity_origin"] != record["target"]["runtime_incarnation"]: + if ( + runtime is None + or runtime["identity_origin"] != record["target"]["runtime_incarnation"] + or record["target"]["root_instance_id"] != checkpoint["root_instance_id"] + ): raise EffectError("effect_not_outstanding") + event_id = hash_value( + ["determa-effect-result-event-1", record["effect_id"], mapping["result_slot"]] + ) envelope = { "event": mapping["event"], - "event_id": record["result_event_id"], - "cause_id": record["result_event_id"], + "event_id": event_id, + "cause_id": event_id, "source": {"host": True}, - "target": runtime["target_identity"], - "payload": record["outcome"]["payload"], + "target": copy.deepcopy(runtime["target_identity"]), + "payload": copy.deepcopy(payload), } location = mapping["operation_token_location"] - if location is not None and location["kind"] == "correlation_id": - envelope["correlation_id"] = record["operation_token"] + if location is not None: + if location["kind"] == "correlation_id": + envelope["correlation_id"] = record["operation_token"] + else: + try: + logical = decoded_typed_value(payload) + pointer = location["pointer"] + if not pointer.startswith("/") or re.search(r"~(?![01])", pointer): + raise ValueError("invalid token pointer") + parts = [ + part.replace("~1", "/").replace("~0", "~") + for part in pointer[1:].split("/") + ] + parent = logical + for part in parts[:-1]: + parent = parent[part] + if not isinstance(parent, dict) or ( + parts[-1] in parent and not isinstance(parent[parts[-1]], str) + ): + raise ValueError("token field must be a string") + parent[parts[-1]] = record["operation_token"] + envelope["payload"] = typed_value(logical) + except (KeyError, TypeError, ValueError) as exc: + raise EffectError("invalid_host_request") from exc from .queueing import _entry_digest delivery = { @@ -956,8 +1089,25 @@ def _admit(self, document: dict[str, Any], record: dict[str, Any]) -> None: "envelope": envelope, "envelope_digest": _entry_digest(checkpoint["root_instance_id"], "input", envelope), } + try: + _preflight_checkpoint_admission( + checkpoint, + [delivery], + self.resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + except ArtifactError as exc: + raise EffectError("invalid_host_request") from exc + return delivery + + def _admit(self, document: dict[str, Any], record: dict[str, Any]) -> None: + checkpoint = document["checkpoint"] + delivery = self._result_delivery( + checkpoint, record, record["outcome"]["kind"], record["outcome"]["payload"] + ) if self.core_observer is not None: - self.core_observer("admit", record["result_event_id"], runtime["target_identity"]) + self.core_observer("admit", record["result_event_id"], delivery["envelope"]["target"]) candidate = admit_checkpoint_v1( checkpoint, [delivery], @@ -969,6 +1119,10 @@ def _admit(self, document: dict[str, Any], record: dict[str, Any]) -> None: record["admission_receipt"] = copy.deepcopy(candidate["operation_receipts"][-1]) record["invocation_state"] = "result_admitted" _bump(document["journal"], candidate) + key = record["effect_id"] + ":" + record["outcome"]["attempt_fence"] + document.setdefault("result_responses", {})[key] = _result_response( + record, document["journal"], candidate, "committed" + ) def recover(self, root: str) -> dict[str, Any]: def change(document: dict[str, Any]) -> dict[str, Any]: @@ -1060,6 +1214,9 @@ def change(document: dict[str, Any]) -> dict[str, Any]: return replay if record["invocation_state"] == "unclaimed" and record["attempt_fence"] == "0": mapping = _mapping(record, "cancelled") + self._result_delivery( + document["checkpoint"], record, "cancelled", request["payload"] + ) state = "prevented_start" outcome = { "kind": "cancelled", diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py new file mode 100644 index 0000000..ba11342 --- /dev/null +++ b/tests/test_committed_effects.py @@ -0,0 +1,128 @@ +"""Native effect outcome and replay regressions against the pinned v1 fixtures.""" + +import copy +import json + +import pytest + +from conformance.harness import conformance_root +from determa.state import MemoryArtifactResolver, load_bundle +from determa.state.effects import ( + EffectError, + SQLiteCommittedEffectHost, + seal_journal, + validate_journal, +) + +CASE = conformance_root() / "conformance/profiles/committed-native-effects/effect-01-result" + + +def read(name): + return json.loads((CASE / name).read_text()) + + +def host_fixture(tmp_path, observer=None): + bundle = load_bundle((CASE / "machine.yaml").read_text()) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + host = SQLiteCommittedEffectHost( + tmp_path / "effects.sqlite", resolver, {}, lambda *_: {}, core_observer=observer + ) + host.setup_schema() + checkpoint = read("pending-checkpoint.json") + claim = read("data/active-claim.json") + host.seed(checkpoint, read("data/leased-journal.json"), claim) + context = { + "principal": claim["worker_principal"], + "scope": claim["scope_identity"], + "epoch": claim["scope_authority_epoch"], + "trusted_now": "0", + } + return host, checkpoint["root_instance_id"], read("data/result-request.json"), context + + +def test_admission_failure_keeps_outcome_durable_and_recovery_admits_once(tmp_path): + def fail(*_): + raise RuntimeError("crash before admission") + + host, root, request, context = host_fixture(tmp_path, fail) + with pytest.raises(RuntimeError, match="crash before admission"): + host.submit_result(root, request, **context) + saved = host.snapshot(root) + record = saved["journal"]["effect_records"][0] + assert record["invocation_state"] == "outcome_recorded" + assert record["outcome"]["payload"] == request["payload"] + assert saved["journal"]["journal_revision"] == "3" + calls = [] + host.core_observer = lambda *args: calls.append(args) + recovered = host.recover(root) + assert recovered["journal"]["effect_records"][0]["invocation_state"] == "result_admitted" + assert len(calls) == 1 + host.recover(root) + assert len(calls) == 1 + + +def test_equal_result_replay_retains_first_response_after_outbox_change(tmp_path): + host, root, request, context = host_fixture(tmp_path) + first = host.submit_result(root, request, **context) + host.terminalize_outbox(root, request["effect_id"], {"status": "confirmed"}) + before = host.snapshot(root) + assert host.submit_result(root, request, **context) == first + assert host.snapshot(root) == before + + +def test_terminal_outcome_requires_immutable_attempt_evidence(): + journal = read("data/outcome-recorded-journal.json") + journal["effect_records"][0]["attempt_records"] = [] + with pytest.raises(EffectError, match="invalid_effect_journal"): + validate_journal(read("pending-checkpoint.json"), seal_journal(journal)) + + +@pytest.mark.parametrize("field", ["digest", "attempt_fence"]) +def test_terminal_outcome_rejects_forged_evidence(field): + journal = copy.deepcopy(read("data/outcome-recorded-journal.json")) + journal["effect_records"][0]["outcome"][field] = ( + "sha256:" + "0" * 64 if field == "digest" else "0" + ) + with pytest.raises(EffectError, match="invalid_effect_journal"): + validate_journal(read("pending-checkpoint.json"), seal_journal(journal)) + + +def test_invalid_result_payload_is_rejected_before_journal_or_core_mutation(tmp_path): + calls = [] + host, root, request, context = host_fixture(tmp_path, lambda *args: calls.append(args)) + request["payload"] = ["map", [["provider_reference", ["int", "42"]]]] + before = host.snapshot(root) + assert host.submit_result(root, request, **context)["status"] == "rejected" + assert host.snapshot(root) == before + assert calls == [] + + +def test_payload_pointer_inserts_pinned_token_before_declared_input_admission(tmp_path): + bundle = load_bundle((CASE / "machine.yaml").read_text()) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + host = SQLiteCommittedEffectHost(tmp_path / "pointer.sqlite", resolver, {}, lambda *_: {}) + host.setup_schema() + checkpoint = read("pending-checkpoint.json") + journal = read("data/leased-journal.json") + journal["effect_records"][0]["result_mapping"][0]["operation_token_location"] = { + "kind": "payload", + "pointer": "/provider_reference", + } + claim = read("data/active-claim.json") + host.seed(checkpoint, seal_journal(journal), claim) + request = read("data/result-request.json") + response = host.submit_result( + checkpoint["root_instance_id"], + request, + principal=claim["worker_principal"], + scope=claim["scope_identity"], + epoch=claim["scope_authority_epoch"], + trusted_now="0", + ) + assert response["status"] == "committed" + saved = host.snapshot(checkpoint["root_instance_id"]) + runtime = saved["checkpoint"]["root_record"]["aggregate_state"]["runtimes"][0] + assert runtime["ready_mailbox"][-1]["envelope"]["payload"] == [ + "map", + [["provider_reference", ["string", request["operation_token"]]]], + ] From ab045d377478740d10616676dd80088db7fe95d5 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 05:31:47 +0000 Subject: [PATCH 05/30] Track effect checkpoint provenance in native authority inventory Preserve other roots worker claims and project private provenance out of closed worker observations. Twenty-two focused native tests passed. Ruff, formatting and mypy now pass, correcting the ArtifactError import type-check failure reported after the preceding checkpoint was queued. Full effects validation and independent review remain unfinished. --- conformance/authority_adapter.py | 2 +- conformance/effects_adapter.py | 1 + src/determa/state/authority.py | 56 ++++++++++++++++++++++---------- src/determa/state/effects.py | 22 +++++++++++-- 4 files changed, 61 insertions(+), 20 deletions(-) diff --git a/conformance/authority_adapter.py b/conformance/authority_adapter.py index dc1896f..134db61 100644 --- a/conformance/authority_adapter.py +++ b/conformance/authority_adapter.py @@ -416,7 +416,7 @@ def run(payload: dict[str, Any]) -> dict[str, Any]: return { "binding": _binding(), "accepted": accepted, - "ledger_after": authority.inspect("scope-42"), + "ledger_after": _observed_ledger(authority), "host_mutation_count": 0, "external_dispatch_count": len(dispatched), } diff --git a/conformance/effects_adapter.py b/conformance/effects_adapter.py index 1ae58df..5ff40d5 100644 --- a/conformance/effects_adapter.py +++ b/conformance/effects_adapter.py @@ -120,6 +120,7 @@ def _authority_ledger() -> dict[str, Any]: "receipts": [], "mutation_bytes": [], "checkpoint_bytes": [], + "native_checkpoint_bytes": [], "journal_entries": [], "ingress_acknowledgements": [], "active_claims": [], diff --git a/src/determa/state/authority.py b/src/determa/state/authority.py index d16d56c..4502e5b 100644 --- a/src/determa/state/authority.py +++ b/src/determa/state/authority.py @@ -299,22 +299,42 @@ def _native_checkpoint_history(ledger: Mapping[str, Any]) -> list[str]: def _native_checkpoints( connection: sqlite3.Connection, ledger: Mapping[str, Any] ) -> list[dict[str, Any]]: - """Read the entire dedicated checkpoint table under the freeze transaction.""" - if ( - connection.execute( - "SELECT 1 FROM sqlite_master WHERE type = 'table' " - "AND name = 'determa_execution_checkpoints'" - ).fetchone() - is None - ): - return [] - documents = [] - for root, payload in connection.execute( - "SELECT root_instance_id, checkpoint FROM determa_execution_checkpoints " - "ORDER BY root_instance_id" - ): + """Read every native checkpoint/journal row under the freeze transaction.""" + documents: dict[str, dict[str, Any]] = {} + history = _native_checkpoint_history(ledger) + tables = { + row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type = 'table'") + } + rows: list[tuple[str, bytes]] = [] + if "determa_execution_checkpoints" in tables: + rows.extend( + connection.execute( + "SELECT root_instance_id, checkpoint FROM determa_execution_checkpoints " + "ORDER BY root_instance_id" + ) + ) + if "determa_committed_effects" in tables: + from .effects import validate_journal + from .wire import canonical_bytes + + for root, payload in connection.execute( + "SELECT root_instance_id, document FROM determa_committed_effects " + "ORDER BY root_instance_id" + ): + native = _parse(bytes(payload).decode("utf-8", "strict")) + validate_journal(native["checkpoint"], native["journal"]) + if native["journal"]["scope_identity"] != ledger["scope_identity"]: + raise ValueError("untracked native effect scope") + for record in native["journal"]["effect_records"]: + if { + "work_identity": record["effect_id"], + "attempt_fence": record["attempt_fence"], + } not in ledger["journal_entries"]: + raise ValueError("untracked native journal") + rows.append((root, canonical_bytes(native["checkpoint"]))) + for root, payload in rows: source = bytes(payload).decode("utf-8", "strict") - if root not in ledger["roots"] or source not in _native_checkpoint_history(ledger): + if root not in ledger["roots"] or source not in history: raise ValueError("untracked native checkpoint") document = _parse(source) if not validate_execution_checkpoint_member("executionCheckpoint", document): @@ -323,8 +343,10 @@ def _native_checkpoints( "execution_checkpoint_digest" ] != execution_checkpoint_digest(document): raise ValueError("native checkpoint identity mismatch") - documents.append(document) - return documents + if root in documents and documents[root] != document: + raise ValueError("divergent native checkpoint owners") + documents[root] = document + return list(documents.values()) def _compose_authority_profile( diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 0911c59..611375f 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -22,9 +22,9 @@ restore_execution_checkpoint_v1, step_checkpoint_v1, ) +from .errors import ArtifactError from .host import outbox_intent_digest from .wire import ( - ArtifactError, ArtifactResolver, _schema_registry, canonical_bytes, @@ -462,12 +462,21 @@ def _mirror_authority(ledger: dict[str, Any], document: dict[str, Any]) -> None: ) else: entry["attempt_fence"] = record["attempt_fence"] + work = {record["effect_id"] for record in records} ledger["active_claims"] = [ + claim for claim in ledger["active_claims"] if claim["work_identity"] not in work + ] + [ copy.deepcopy(document["claims"][record["effect_id"]]) for record in records if record["invocation_state"] == "leased" and record["effect_id"] in document["claims"] ] - ledger["checkpoint_bytes"].append(canonical_bytes(document["checkpoint"]).decode()) + from .authority import _native_checkpoint_history + + source = canonical_bytes(document["checkpoint"]).decode() + history = _native_checkpoint_history(ledger) + if source not in history: + history.append(source) + ledger["checkpoint_bytes"].append(source) ledger["mutation_bytes"].append(canonical_bytes(document["journal"]).decode()) ledger["scope_generation"] = str(int(ledger["scope_generation"]) + 1) @@ -512,6 +521,15 @@ def _transact( if authority is None: raise EffectError("stale_scope_authority") ledger = json.loads(authority[0]) + from .authority import _native_checkpoint_history + + if ( + root not in ledger["roots"] + or document["journal"]["scope_identity"] != self.authority_scope + or canonical_bytes(document["checkpoint"]).decode() + not in _native_checkpoint_history(ledger) + ): + raise EffectError("unauthorized_scope") if ledger["state"] != "active" or ( expected_epoch is not None and ledger["authority_epoch"] != expected_epoch ): From 1cf80a7d68fa90ff74131fa8caba2f3e7402cb0a Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 05:40:47 +0000 Subject: [PATCH 06/30] Guard effect bootstrap and separate expired workers from host recovery Twelve native regressions, Ruff, formatting and mypy pass. Seed now rejects frozen/stale scopes and unissued claims; conformance setup explicitly installs pre-existing fixture authority facts. Full profile reached effects after authority passed, then exposed the corrected expired-worker case. Full validation and independent review remain unfinished. --- conformance/effects_adapter.py | 19 ++++++++++ src/determa/state/effects.py | 33 +++++++++++++++-- tests/test_committed_effects.py | 66 +++++++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 3 deletions(-) diff --git a/conformance/effects_adapter.py b/conformance/effects_adapter.py index 5ff40d5..049f146 100644 --- a/conformance/effects_adapter.py +++ b/conformance/effects_adapter.py @@ -319,6 +319,25 @@ def observe_core(kind: str, event_id: str, target: dict[str, Any]) -> None: authority_scope=_AUTHORITY_SCOPE, core_observer=observe_core, ) + # Install the vector's pre-existing authority facts as trusted test setup. + # The production seed path may restore these claims, but cannot issue them. + with authority._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + "SELECT ledger FROM determa_scope_authority WHERE scope_identity = ?", + (_AUTHORITY_SCOPE,), + ).fetchone() + ledger = json.loads(row[0]) + ledger["active_claims"] = [copy.deepcopy(payload["claim"])] if payload["claim"] else [] + ledger["journal_entries"] = [ + {"work_identity": record["effect_id"], "attempt_fence": record["attempt_fence"]} + for record in journal["effect_records"] + ] + connection.execute( + "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", + (json.dumps(ledger), _AUTHORITY_SCOPE), + ) + connection.commit() host.seed(checkpoint, journal, payload["claim"]) before = host.snapshot(_ROOT) operation = payload["operation"] diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 611375f..2d5154d 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -431,8 +431,35 @@ def seed( if row is None: raise EffectError("stale_scope_authority") ledger = json.loads(row[0]) - if checkpoint["root_instance_id"] not in ledger["roots"]: + if ( + ledger["state"] != "active" + or self.route.get("authority_epoch") != ledger["authority_epoch"] + ): + raise EffectError("stale_scope_authority") + if ( + checkpoint["root_instance_id"] not in ledger["roots"] + or journal["scope_identity"] != self.authority_scope + ): raise EffectError("unauthorized_scope") + # Restoring an already claimed invocation cannot mint authority. + # Its exact claim must already have been issued by this ledger. + if claim is not None and ( + dict(claim) not in ledger["active_claims"] + or claim.get("scope_identity") != self.authority_scope + or claim.get("root_instance_id") != checkpoint["root_instance_id"] + or claim.get("scope_authority_epoch") != ledger["authority_epoch"] + ): + raise EffectError("stale_attempt_fence") + for record in journal["effect_records"]: + if ( + record["attempt_fence"] != "0" + and { + "work_identity": record["effect_id"], + "attempt_fence": record["attempt_fence"], + } + not in ledger["journal_entries"] + ): + raise EffectError("stale_attempt_fence") self._mirror_authority(ledger, document) connection.execute( "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", @@ -544,7 +571,6 @@ def _transact( raise EffectError("unauthorized_scope") replay = journal_record["invocation_state"] in { "result_admitted", - "outcome_recorded", "closed", "unclaimed", "ambiguous", @@ -963,7 +989,8 @@ def _submit( scope, epoch, trusted_now, - allow_replay=retained is not None, + allow_replay=retained is not None + and (kind not in _OUTCOMES or preexisting["admission_receipt"] is not None), ) key = effect_id + ":" + fence prior = document["result_requests"].get(key) diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index ba11342..6fef1c7 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -126,3 +126,69 @@ def test_payload_pointer_inserts_pinned_token_before_declared_input_admission(tm "map", [["provider_reference", ["string", request["operation_token"]]]], ] + + +def test_expired_worker_cannot_admit_recorded_outcome_but_host_recovery_can(tmp_path): + host, root, request, context = host_fixture(tmp_path) + host.submit_result(root, request, admit=False, **context) + before = host.snapshot(root) + context["trusted_now"] = read("data/active-claim.json")["expires_at"] + assert host.submit_result(root, request, **context)["error_code"] == "stale_attempt_fence" + assert host.snapshot(root) == before + assert ( + host.recover(root)["journal"]["effect_records"][0]["invocation_state"] == "result_admitted" + ) + + +@pytest.mark.parametrize("boundary", ["frozen", "wrong_epoch", "wrong_scope", "unissued_claim"]) +def test_authority_seed_cannot_bypass_scope_or_claim_guard(tmp_path, boundary): + from determa.state.authority import SQLiteLocalAuthority + from determa.state.wire import hash_value + + bundle = load_bundle((CASE / "machine.yaml").read_text()) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + checkpoint = read("pending-checkpoint.json") + journal = read("data/leased-journal.json") + claim = read("data/active-claim.json") + scope, root = journal["scope_identity"], checkpoint["root_instance_id"] + authority = SQLiteLocalAuthority(tmp_path / "authority.sqlite") + authority.setup_schema() + assert authority.allocate(scope, "owner", roots=(root,)) + claim["scope_authority_epoch"] = "0" + epoch = "9" if boundary == "wrong_epoch" else "0" + host = SQLiteCommittedEffectHost( + authority.path, resolver, {"authority_epoch": epoch}, lambda *_: {}, authority_scope=scope + ) + host.setup_schema() + if boundary == "frozen": + request = { + "interface": "determa.host_authority", + "interface_version": 1, + "operation": "freeze_scope", + "operation_id": "freeze", + "scope_identity": scope, + "expected_authority_epoch": "0", + "expected_scope_generation": "0", + "arguments": {}, + } + request["request_digest"] = hash_value(["determa-host-authority-request-1", request]) + response = authority.perform( + json.dumps(request), + { + "authenticated_principal": "owner", + "authorized_scopes": [scope], + "operation_rights": ["freeze_scope"], + }, + ) + assert json.loads(response)["state"] == "frozen" + if boundary == "wrong_scope": + journal["scope_identity"] = "other-scope" + journal = seal_journal(journal) + before = authority.inspect(scope) + with pytest.raises(EffectError): + host.seed(checkpoint, journal, claim) + assert authority.inspect(scope) == before + with host._connect() as connection: + assert ( + connection.execute("SELECT COUNT(*) FROM determa_committed_effects").fetchone()[0] == 0 + ) From 85c9840a4aeb8499df839bd2b004d2d31efe0043 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 05:48:27 +0000 Subject: [PATCH 07/30] Preserve issued worker fences during effect bootstrap Reject older zero-fence snapshots and removal of live issued claims. Fourteen native regressions, Ruff, formatting and mypy pass. All 43 effects-only diagnostic vectors passed at the prior checkpoint; final composed authority/effects run and independent review remain required. --- src/determa/state/effects.py | 27 ++++++++++++++++++------- tests/test_committed_effects.py | 36 +++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 2d5154d..31b3a4b 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -451,13 +451,26 @@ def seed( ): raise EffectError("stale_attempt_fence") for record in journal["effect_records"]: - if ( - record["attempt_fence"] != "0" - and { - "work_identity": record["effect_id"], - "attempt_fence": record["attempt_fence"], - } - not in ledger["journal_entries"] + expected = { + "work_identity": record["effect_id"], + "attempt_fence": record["attempt_fence"], + } + issued = next( + ( + item + for item in ledger["journal_entries"] + if item["work_identity"] == record["effect_id"] + ), + None, + ) + if (issued is not None and issued != expected) or ( + issued is None and record["attempt_fence"] != "0" + ): + raise EffectError("stale_attempt_fence") + if any( + item["work_identity"] == record["effect_id"] + and document["claims"].get(record["effect_id"]) != item + for item in ledger["active_claims"] ): raise EffectError("stale_attempt_fence") self._mirror_authority(ledger, document) diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 6fef1c7..977874d 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -192,3 +192,39 @@ def test_authority_seed_cannot_bypass_scope_or_claim_guard(tmp_path, boundary): assert ( connection.execute("SELECT COUNT(*) FROM determa_committed_effects").fetchone()[0] == 0 ) + + +@pytest.mark.parametrize("journal_name", ["unclaimed-journal.json", "leased-journal.json"]) +def test_seed_cannot_rewind_an_issued_fence_or_remove_its_claim(tmp_path, journal_name): + from determa.state.authority import SQLiteLocalAuthority + + bundle = load_bundle((CASE / "machine.yaml").read_text()) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + checkpoint = read("pending-checkpoint.json") + journal = read("data/" + journal_name) + claim = read("data/active-claim.json") + claim["scope_authority_epoch"] = "0" + scope, root = journal["scope_identity"], checkpoint["root_instance_id"] + authority = SQLiteLocalAuthority(tmp_path / "issued.sqlite") + authority.setup_schema() + assert authority.allocate(scope, "owner", roots=(root,)) + ledger = authority.inspect(scope) + ledger["active_claims"] = [claim] + ledger["journal_entries"] = [{"work_identity": claim["work_identity"], "attempt_fence": "1"}] + with authority._connect() as connection: + connection.execute( + "UPDATE determa_scope_authority SET ledger=? WHERE scope_identity=?", + (json.dumps(ledger), scope), + ) + connection.commit() + host = SQLiteCommittedEffectHost( + authority.path, resolver, {"authority_epoch": "0"}, lambda *_: {}, authority_scope=scope + ) + host.setup_schema() + with pytest.raises(EffectError, match="stale_attempt_fence"): + host.seed(checkpoint, journal) + assert authority.inspect(scope) == ledger + with host._connect() as connection: + assert ( + connection.execute("SELECT COUNT(*) FROM determa_committed_effects").fetchone()[0] == 0 + ) From 5ee1aee9e766717d9bd248bf85e48bc97a3c0fcc Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 05:59:58 +0000 Subject: [PATCH 08/30] Checkpoint atomic authority and native effect claims --- conformance/authority_adapter.py | 6 +- conformance/effects_adapter.py | 14 ++- src/determa/state/authority.py | 77 +++++++++++++++- src/determa/state/effects.py | 150 ++++++++++++++++++++++++------- tests/test_committed_effects.py | 142 ++++++++++++++++++++++++++++- 5 files changed, 349 insertions(+), 40 deletions(-) diff --git a/conformance/authority_adapter.py b/conformance/authority_adapter.py index 134db61..21f8176 100644 --- a/conformance/authority_adapter.py +++ b/conformance/authority_adapter.py @@ -167,7 +167,11 @@ def _observed_ledger(authority: SQLiteLocalAuthority) -> dict[str, Any] | None: if ledger is None: return None # Project the actual ledger onto the closed conformance observation shape. - return {key: value for key, value in ledger.items() if key != "native_checkpoint_bytes"} + return { + key: value + for key, value in ledger.items() + if key not in {"native_checkpoint_bytes", "effect_claim_history"} + } def _call(authority: SQLiteLocalAuthority, call: dict[str, Any], **kwargs: Any) -> str | None: diff --git a/conformance/effects_adapter.py b/conformance/effects_adapter.py index 049f146..7bdb727 100644 --- a/conformance/effects_adapter.py +++ b/conformance/effects_adapter.py @@ -328,7 +328,19 @@ def observe_core(kind: str, event_id: str, target: dict[str, Any]) -> None: (_AUTHORITY_SCOPE,), ).fetchone() ledger = json.loads(row[0]) - ledger["active_claims"] = [copy.deepcopy(payload["claim"])] if payload["claim"] else [] + ledger["effect_claim_history"] = ( + [copy.deepcopy(payload["claim"])] if payload["claim"] else [] + ) + ledger["active_claims"] = ( + [copy.deepcopy(payload["claim"])] + if payload["claim"] + and any( + record["invocation_state"] == "leased" + and record["effect_id"] == payload["claim"]["work_identity"] + for record in journal["effect_records"] + ) + else [] + ) ledger["journal_entries"] = [ {"work_identity": record["effect_id"], "attempt_fence": record["attempt_fence"]} for record in journal["effect_records"] diff --git a/src/determa/state/authority.py b/src/determa/state/authority.py index 4502e5b..5037ba7 100644 --- a/src/determa/state/authority.py +++ b/src/determa/state/authority.py @@ -934,7 +934,18 @@ def _perform( if ( entry is None or arguments["root_instance_id"] not in ledger["roots"] - or entry["attempt_fence"] != arguments["expected_attempt_fence"] + or ( + entry["attempt_fence"] != arguments["expected_attempt_fence"] + and not ( + arguments["expected_attempt_fence"] is None + and entry["attempt_fence"] == "0" + and not any( + claim["work_identity"] == arguments["work_identity"] + for claim in ledger["active_claims"] + + ledger.get("effect_claim_history", []) + ) + ) + ) ): return _compact(_result(request, ledger, "stale_attempt_fence")) if invocation.get("assigned_worker_principal") != principal: @@ -952,6 +963,70 @@ def _perform( "expires_at": str(int(now) + self.worker_lease_nanoseconds), "state": "active", } + # An installed effect journal is a participant in this claim, + # not an eventually updated mirror of the authority receipt. + if connection.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' " + "AND name='determa_committed_effects'" + ).fetchone(): + from .effects import ( + EffectError, + SQLiteCommittedEffectHost, + _issue_effect_claim, + _validate_authority_pair, + validate_journal, + ) + from .wire import canonical_bytes + + native_rows = connection.execute( + "SELECT root_instance_id, document FROM determa_committed_effects" + ).fetchall() + for native_root, native_bytes in native_rows: + native = _parse(bytes(native_bytes).decode()) + found = next( + ( + record + for record in native["journal"]["effect_records"] + if record["effect_id"] == arguments["work_identity"] + ), + None, + ) + if found is None: + continue + if ( + native_root != arguments["root_instance_id"] + or found["operation_token"] != arguments["operation_token"] + ): + return _compact(_result(request, ledger, "invalid_host_request")) + try: + validate_journal(native["checkpoint"], native["journal"]) + _validate_authority_pair(ledger, native) + issued = _issue_effect_claim( + native, + arguments["work_identity"], + principal, + ledger["authority_epoch"], + new_claim["expires_at"], + now, + ) + if issued != new_claim: + raise EffectError("stale_attempt_fence") + validate_journal(native["checkpoint"], native["journal"]) + SQLiteCommittedEffectHost._mirror_authority( + ledger, native, advance_generation=False + ) + except EffectError as error: + code = ( + "stale_attempt_fence" + if error.code == "stale_attempt_fence" + else "host_capability_mismatch" + ) + return _compact(_result(request, ledger, code)) + connection.execute( + "UPDATE determa_committed_effects SET document=? " + "WHERE root_instance_id=?", + (canonical_bytes(native), native_root), + ) entry["attempt_fence"] = fence ledger["active_claims"] = [ item diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 31b3a4b..7421d35 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -359,6 +359,87 @@ def _reconstruct_producer_response( return response +def _validate_authority_pair(ledger: Mapping[str, Any], document: Mapping[str, Any]) -> None: + """Check the old native pair before any journal transition can be staged.""" + journal = document["journal"] + if ( + journal["scope_identity"] != ledger["scope_identity"] + or journal["root_instance_id"] not in ledger["roots"] + ): + raise EffectError("unauthorized_scope") + for record in journal["effect_records"]: + issued = next( + ( + entry + for entry in ledger["journal_entries"] + if entry["work_identity"] == record["effect_id"] + ), + None, + ) + if issued is None or issued["attempt_fence"] != record["attempt_fence"]: + raise EffectError("stale_attempt_fence") + if any( + claim["work_identity"] == record["effect_id"] + and int(claim["attempt_fence"]) > int(record["attempt_fence"]) + for claim in ledger.get("effect_claim_history", []) + ): + raise EffectError("stale_attempt_fence") + live = next( + ( + claim + for claim in ledger["active_claims"] + if claim["work_identity"] == record["effect_id"] + ), + None, + ) + if record["invocation_state"] == "leased": + if live is None or live != document["claims"].get(record["effect_id"]): + raise EffectError("stale_attempt_fence") + elif live is not None: + raise EffectError("stale_attempt_fence") + + +def _issue_effect_claim( + document: dict[str, Any], + effect_id: str, + principal: str, + epoch: str, + expires_at: str, + trusted_now: str, + *, + deduplication_proven: bool = False, +) -> dict[str, Any]: + now, expiry = _now(trusted_now), _now(expires_at) + if expiry <= now: + raise EffectError("stale_attempt_fence") + journal = document["journal"] + record = _record(journal, effect_id) + if record["invocation_state"] not in {"unclaimed", "ambiguous"}: + raise EffectError("effect_not_outstanding") + if record["invocation_state"] == "ambiguous" and not ( + deduplication_proven and record["idempotency_policy"] == "destination_deduplicates" + ): + raise EffectError("effect_not_outstanding") + fence = str(int(record["attempt_fence"]) + 1) + claim = { + "scope_identity": journal["scope_identity"], + "root_instance_id": journal["root_instance_id"], + "work_kind": "effect", + "work_identity": effect_id, + "operation_token": record["operation_token"], + "scope_authority_epoch": epoch, + "attempt_fence": fence, + "worker_principal": principal, + "expires_at": expires_at, + "state": "active", + } + record["attempt_fence"] = fence + record["invocation_state"] = "leased" + document["claims"][effect_id] = claim + _bump(journal) + return copy.deepcopy(claim) + + class SQLiteCommittedEffectHost: """One local host with checkpoint, journal, claims and responses in one SQLite row. @@ -444,7 +525,8 @@ def seed( # Restoring an already claimed invocation cannot mint authority. # Its exact claim must already have been issued by this ledger. if claim is not None and ( - dict(claim) not in ledger["active_claims"] + dict(claim) + not in ledger["active_claims"] + ledger.get("effect_claim_history", []) or claim.get("scope_identity") != self.authority_scope or claim.get("root_instance_id") != checkpoint["root_instance_id"] or claim.get("scope_authority_epoch") != ledger["authority_epoch"] @@ -467,9 +549,20 @@ def seed( issued is None and record["attempt_fence"] != "0" ): raise EffectError("stale_attempt_fence") + if ( + record["invocation_state"] == "leased" + and document["claims"].get(record["effect_id"]) + not in ledger["active_claims"] + ): + raise EffectError("stale_attempt_fence") if any( item["work_identity"] == record["effect_id"] - and document["claims"].get(record["effect_id"]) != item + and ( + document["claims"].get(record["effect_id"]) != item + or record["invocation_state"] != "leased" + or item["attempt_fence"] != record["attempt_fence"] + or item["operation_token"] != record["operation_token"] + ) for item in ledger["active_claims"] ): raise EffectError("stale_attempt_fence") @@ -484,7 +577,9 @@ def seed( connection.commit() @staticmethod - def _mirror_authority(ledger: dict[str, Any], document: dict[str, Any]) -> None: + def _mirror_authority( + ledger: dict[str, Any], document: dict[str, Any], *, advance_generation: bool = True + ) -> None: """Mirror effect fences and exact committed bytes in the colocated authority ledger.""" records = document["journal"]["effect_records"] for record in records: @@ -501,6 +596,8 @@ def _mirror_authority(ledger: dict[str, Any], document: dict[str, Any]) -> None: {"work_identity": record["effect_id"], "attempt_fence": record["attempt_fence"]} ) else: + if int(record["attempt_fence"]) < int(entry["attempt_fence"]): + raise EffectError("stale_attempt_fence") entry["attempt_fence"] = record["attempt_fence"] work = {record["effect_id"] for record in records} ledger["active_claims"] = [ @@ -510,6 +607,10 @@ def _mirror_authority(ledger: dict[str, Any], document: dict[str, Any]) -> None: for record in records if record["invocation_state"] == "leased" and record["effect_id"] in document["claims"] ] + history_claims = ledger.setdefault("effect_claim_history", []) + for claim in document["claims"].values(): + if claim not in history_claims: + history_claims.append(copy.deepcopy(claim)) from .authority import _native_checkpoint_history source = canonical_bytes(document["checkpoint"]).decode() @@ -518,7 +619,8 @@ def _mirror_authority(ledger: dict[str, Any], document: dict[str, Any]) -> None: history.append(source) ledger["checkpoint_bytes"].append(source) ledger["mutation_bytes"].append(canonical_bytes(document["journal"]).decode()) - ledger["scope_generation"] = str(int(ledger["scope_generation"]) + 1) + if advance_generation: + ledger["scope_generation"] = str(int(ledger["scope_generation"]) + 1) def snapshot(self, root_instance_id: str) -> dict[str, Any]: with self._connect() as connection: @@ -574,6 +676,7 @@ def _transact( expected_epoch is not None and ledger["authority_epoch"] != expected_epoch ): raise EffectError("stale_scope_authority") + _validate_authority_pair(ledger, document) if worker_guard is not None: effect_id, principal, fence, trusted_now = worker_guard journal_record = _record(document["journal"], effect_id) @@ -753,37 +856,16 @@ def claim( trusted_now: str, deduplication_proven: bool = False, ) -> dict[str, Any]: - now, expiry = _now(trusted_now), _now(expires_at) - if expiry <= now: - raise EffectError("stale_attempt_fence") - def change(document: dict[str, Any]) -> dict[str, Any]: - journal = document["journal"] - record = _record(journal, effect_id) - if record["invocation_state"] not in {"unclaimed", "ambiguous"}: - raise EffectError("effect_not_outstanding") - if record["invocation_state"] == "ambiguous" and not ( - deduplication_proven and record["idempotency_policy"] == "destination_deduplicates" - ): - raise EffectError("effect_not_outstanding") - fence = str(int(record["attempt_fence"]) + 1) - claim = { - "scope_identity": journal["scope_identity"], - "root_instance_id": root, - "work_kind": "effect", - "work_identity": effect_id, - "operation_token": record["operation_token"], - "scope_authority_epoch": epoch, - "attempt_fence": fence, - "worker_principal": principal, - "expires_at": expires_at, - "state": "active", - } - record["attempt_fence"] = fence - record["invocation_state"] = "leased" - document["claims"][effect_id] = claim - _bump(journal) - return copy.deepcopy(claim) + return _issue_effect_claim( + document, + effect_id, + principal, + epoch, + expires_at, + trusted_now, + deduplication_proven=deduplication_proven, + ) return self._transact( root, change, expected_epoch=epoch, authority_mutation=self._mirror_authority diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 977874d..1c51c6c 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -194,8 +194,15 @@ def test_authority_seed_cannot_bypass_scope_or_claim_guard(tmp_path, boundary): ) -@pytest.mark.parametrize("journal_name", ["unclaimed-journal.json", "leased-journal.json"]) -def test_seed_cannot_rewind_an_issued_fence_or_remove_its_claim(tmp_path, journal_name): +@pytest.mark.parametrize( + "journal_name,with_claim", + [ + ("unclaimed-journal.json", False), + ("leased-journal.json", False), + ("retryable-journal.json", True), + ], +) +def test_seed_cannot_rewind_an_issued_fence_or_remove_its_claim(tmp_path, journal_name, with_claim): from determa.state.authority import SQLiteLocalAuthority bundle = load_bundle((CASE / "machine.yaml").read_text()) @@ -222,9 +229,138 @@ def test_seed_cannot_rewind_an_issued_fence_or_remove_its_claim(tmp_path, journa ) host.setup_schema() with pytest.raises(EffectError, match="stale_attempt_fence"): - host.seed(checkpoint, journal) + host.seed(checkpoint, journal, claim if with_claim else None) assert authority.inspect(scope) == ledger with host._connect() as connection: assert ( connection.execute("SELECT COUNT(*) FROM determa_committed_effects").fetchone()[0] == 0 ) + + +def authority_effect_fixture(tmp_path): + from determa.state.authority import SQLiteLocalAuthority + + bundle = load_bundle((CASE / "machine.yaml").read_text()) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + checkpoint, journal = read("pending-checkpoint.json"), read("data/unclaimed-journal.json") + scope, root = journal["scope_identity"], checkpoint["root_instance_id"] + authority = SQLiteLocalAuthority( + tmp_path / "paired.sqlite", worker_fencing=True, worker_lease_nanoseconds=10 + ) + authority.setup_schema() + assert authority.allocate(scope, "owner", roots=(root,)) + host = SQLiteCommittedEffectHost( + authority.path, resolver, {"authority_epoch": "0"}, lambda *_: {}, authority_scope=scope + ) + host.setup_schema() + host.seed(checkpoint, journal) + return authority, host, scope, root, journal["effect_records"][0] + + +def authority_claim_request( + authority, scope, root, record, operation="claim-first", expected=None, **overrides +): + from determa.state.wire import hash_value + + arguments = { + "root_instance_id": root, + "work_kind": "effect", + "work_identity": record["effect_id"], + "operation_token": record["operation_token"], + "expected_attempt_fence": expected, + "expected_worker_principal": "worker-a", + } + arguments.update(overrides) + request = { + "interface": "determa.host_authority", + "interface_version": 1, + "operation": "fence_worker", + "operation_id": operation, + "scope_identity": scope, + "expected_authority_epoch": "0", + "expected_scope_generation": authority.inspect(scope)["scope_generation"], + "arguments": arguments, + } + request["request_digest"] = hash_value(["determa-host-authority-request-1", request]) + invocation = { + "authenticated_principal": "worker-a", + "authorized_scopes": [scope], + "operation_rights": ["fence_worker"], + "assigned_worker_principal": "worker-a", + "clock_available": True, + "trusted_clock_now": "0", + } + return json.dumps(request), invocation + + +def test_authority_claim_atomically_updates_native_journal_and_replays_once(tmp_path): + authority, host, scope, root, record = authority_effect_fixture(tmp_path) + request, invocation = authority_claim_request(authority, scope, root, record) + before = host.snapshot(root) + response = authority.perform(request, invocation) + result = json.loads(response) + assert result["status"] == "accepted" + after = host.snapshot(root) + assert after["claims"][record["effect_id"]] == result["claim"] + assert after["journal"]["effect_records"][0]["invocation_state"] == "leased" + assert after["journal"]["effect_records"][0]["attempt_fence"] == "1" + assert ( + int(after["journal"]["journal_revision"]) == int(before["journal"]["journal_revision"]) + 1 + ) + ledger = authority.inspect(scope) + assert ledger["active_claims"] == [result["claim"]] + assert authority.perform(request, invocation) == response + assert host.snapshot(root) == after + assert authority.inspect(scope) == ledger + # Lease expiration alone does not authorize another external attempt. + retry, invocation = authority_claim_request(authority, scope, root, record, "unsafe-retry", "1") + invocation["trusted_clock_now"] = "20" + assert json.loads(authority.perform(retry, invocation))["status"] == "rejected" + assert host.snapshot(root) == after + assert authority.inspect(scope) == ledger + + +def test_authority_claim_rolls_back_both_native_records_before_commit(tmp_path): + authority, host, scope, root, record = authority_effect_fixture(tmp_path) + request, invocation = authority_claim_request(authority, scope, root, record) + before, ledger = host.snapshot(root), authority.inspect(scope) + assert authority._perform(request, invocation, fault="precommit_abort") is None + assert host.snapshot(root) == before + assert authority.inspect(scope) == ledger + + +@pytest.mark.parametrize( + "overrides", [{"operation_token": "wrong-token"}, {"root_instance_id": "wrong-root"}] +) +def test_authority_claim_cannot_invent_native_identity(tmp_path, overrides): + authority, host, scope, root, record = authority_effect_fixture(tmp_path) + request, invocation = authority_claim_request(authority, scope, root, record, **overrides) + before, ledger = host.snapshot(root), authority.inspect(scope) + assert json.loads(authority.perform(request, invocation))["status"] == "rejected" + assert host.snapshot(root) == before + assert authority.inspect(scope) == ledger + + +@pytest.mark.parametrize("changed", ["ledger", "journal"]) +def test_effect_mutations_refuse_a_torn_authority_journal_pair(tmp_path, changed): + authority, host, scope, root, record = authority_effect_fixture(tmp_path) + if changed == "ledger": + ledger = authority.inspect(scope) + ledger["journal_entries"][0]["attempt_fence"] = "1" + with authority._connect() as connection: + connection.execute("UPDATE determa_scope_authority SET ledger=?", (json.dumps(ledger),)) + connection.commit() + else: + document = host.snapshot(root) + document["journal"]["effect_records"][0]["attempt_fence"] = "1" + document["journal"] = seal_journal(document["journal"]) + with host._connect() as connection: + connection.execute( + "UPDATE determa_committed_effects SET document=?", (json.dumps(document).encode(),) + ) + connection.commit() + before, ledger = host.snapshot(root), authority.inspect(scope) + with pytest.raises(EffectError, match="stale_attempt_fence"): + host.claim(root, record["effect_id"], "worker-a", "0", expires_at="10", trusted_now="0") + assert host.snapshot(root) == before + assert authority.inspect(scope) == ledger From fd7aaa9c48fe992ef6f5f9afa0f9529b410370fb Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 06:02:54 +0000 Subject: [PATCH 09/30] Reject missing native effect participant evidence before issuing claims --- conformance/authority_adapter.py | 2 +- src/determa/state/authority.py | 19 ++++++++--- src/determa/state/effects.py | 17 ++++++++++ tests/test_committed_effects.py | 58 ++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 6 deletions(-) diff --git a/conformance/authority_adapter.py b/conformance/authority_adapter.py index 21f8176..21cee90 100644 --- a/conformance/authority_adapter.py +++ b/conformance/authority_adapter.py @@ -170,7 +170,7 @@ def _observed_ledger(authority: SQLiteLocalAuthority) -> dict[str, Any] | None: return { key: value for key, value in ledger.items() - if key not in {"native_checkpoint_bytes", "effect_claim_history"} + if key not in {"native_checkpoint_bytes", "effect_claim_history", "native_effect_roots"} } diff --git a/src/determa/state/authority.py b/src/determa/state/authority.py index 5037ba7..807efe2 100644 --- a/src/determa/state/authority.py +++ b/src/determa/state/authority.py @@ -965,6 +965,10 @@ def _perform( } # An installed effect journal is a participant in this claim, # not an eventually updated mirror of the authority receipt. + native_owned = arguments["root_instance_id"] in ledger.get( + "native_effect_roots", [] + ) + native_update = None if connection.execute( "SELECT 1 FROM sqlite_master WHERE type='table' " "AND name='determa_committed_effects'" @@ -993,6 +997,8 @@ def _perform( ) if found is None: continue + if native_update is not None: + return _compact(_result(request, ledger, "host_capability_mismatch")) if ( native_root != arguments["root_instance_id"] or found["operation_token"] != arguments["operation_token"] @@ -1022,11 +1028,14 @@ def _perform( else "host_capability_mismatch" ) return _compact(_result(request, ledger, code)) - connection.execute( - "UPDATE determa_committed_effects SET document=? " - "WHERE root_instance_id=?", - (canonical_bytes(native), native_root), - ) + native_update = (canonical_bytes(native), native_root) + if native_owned and native_update is None: + return _compact(_result(request, ledger, "host_capability_mismatch")) + if native_update is not None: + connection.execute( + "UPDATE determa_committed_effects SET document=? WHERE root_instance_id=?", + native_update, + ) entry["attempt_fence"] = fence ledger["active_claims"] = [ item diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 7421d35..9d8f341 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -361,10 +361,17 @@ def _reconstruct_producer_response( def _validate_authority_pair(ledger: Mapping[str, Any], document: Mapping[str, Any]) -> None: """Check the old native pair before any journal transition can be staged.""" + from .authority import _native_checkpoint_history + + try: + history = _native_checkpoint_history(ledger) + except ValueError as error: + raise EffectError("unauthorized_scope") from error journal = document["journal"] if ( journal["scope_identity"] != ledger["scope_identity"] or journal["root_instance_id"] not in ledger["roots"] + or canonical_bytes(document["checkpoint"]).decode() not in history ): raise EffectError("unauthorized_scope") for record in journal["effect_records"]: @@ -533,6 +540,12 @@ def seed( ): raise EffectError("stale_attempt_fence") for record in journal["effect_records"]: + if any( + previous["work_identity"] == record["effect_id"] + and int(previous["attempt_fence"]) > int(record["attempt_fence"]) + for previous in ledger.get("effect_claim_history", []) + ): + raise EffectError("stale_attempt_fence") expected = { "work_identity": record["effect_id"], "attempt_fence": record["attempt_fence"], @@ -581,6 +594,10 @@ def _mirror_authority( ledger: dict[str, Any], document: dict[str, Any], *, advance_generation: bool = True ) -> None: """Mirror effect fences and exact committed bytes in the colocated authority ledger.""" + native_roots = ledger.setdefault("native_effect_roots", []) + root = document["checkpoint"]["root_instance_id"] + if root not in native_roots: + native_roots.append(root) records = document["journal"]["effect_records"] for record in records: entry = next( diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 1c51c6c..78de134 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -364,3 +364,61 @@ def test_effect_mutations_refuse_a_torn_authority_journal_pair(tmp_path, changed host.claim(root, record["effect_id"], "worker-a", "0", expires_at="10", trusted_now="0") assert host.snapshot(root) == before assert authority.inspect(scope) == ledger + + +@pytest.mark.parametrize("damage", ["provenance", "native_row", "native_table", "work_identity"]) +def test_authority_claim_refuses_missing_native_participant_evidence(tmp_path, damage): + authority, host, scope, root, record = authority_effect_fixture(tmp_path) + if damage == "provenance": + ledger = authority.inspect(scope) + ledger["native_checkpoint_bytes"] = [] + with authority._connect() as connection: + connection.execute("UPDATE determa_scope_authority SET ledger=?", (json.dumps(ledger),)) + connection.commit() + elif damage in {"native_row", "native_table"}: + with host._connect() as connection: + connection.execute( + "DELETE FROM determa_committed_effects" + if damage == "native_row" + else "DROP TABLE determa_committed_effects" + ) + connection.commit() + else: + record = {**record, "effect_id": "unrelated-work"} + ledger = authority.inspect(scope) + ledger["journal_entries"].append({"work_identity": "unrelated-work", "attempt_fence": "0"}) + with authority._connect() as connection: + connection.execute("UPDATE determa_scope_authority SET ledger=?", (json.dumps(ledger),)) + connection.commit() + request, invocation = authority_claim_request(authority, scope, root, record) + ledger = authority.inspect(scope) + before = None if damage in {"native_row", "native_table"} else host.snapshot(root) + assert json.loads(authority.perform(request, invocation))["status"] == "rejected" + assert authority.inspect(scope) == ledger + if before is not None: + assert host.snapshot(root) == before + + +def test_seed_preserves_historical_claim_without_reviving_it(tmp_path): + authority, host, scope, root, record = authority_effect_fixture(tmp_path) + request, invocation = authority_claim_request(authority, scope, root, record) + claim = json.loads(authority.perform(request, invocation))["claim"] + # Model a retained completed attempt whose current journal permits a retry. + journal = read("data/retryable-journal.json") + ledger = authority.inspect(scope) + ledger["active_claims"] = [] + assert claim in ledger["effect_claim_history"] + checkpoint = host.snapshot(root)["checkpoint"] + with authority._connect() as connection: + connection.execute("UPDATE determa_scope_authority SET ledger=?", (json.dumps(ledger),)) + connection.execute("DELETE FROM determa_committed_effects") + connection.commit() + host.seed(checkpoint, journal, claim) + assert authority.inspect(scope)["active_claims"] == [] + assert host.snapshot(root)["claims"][record["effect_id"]] == claim + new_claim = host.claim( + root, record["effect_id"], "worker-b", "0", expires_at="30", trusted_now="20" + ) + assert new_claim["attempt_fence"] == "2" + assert authority.inspect(scope)["active_claims"] == [new_claim] + assert claim in authority.inspect(scope)["effect_claim_history"] From f157dd85d069843d61538ca1bb935f0cf6a5bf09 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 06:05:03 +0000 Subject: [PATCH 10/30] Bind native effect work permanently to its original root and token --- conformance/authority_adapter.py | 8 +++++++- src/determa/state/authority.py | 18 ++++++++++++++++-- src/determa/state/effects.py | 24 ++++++++++++++++++++++++ tests/test_committed_effects.py | 13 +++++++++++++ 4 files changed, 60 insertions(+), 3 deletions(-) diff --git a/conformance/authority_adapter.py b/conformance/authority_adapter.py index 21cee90..4886cc3 100644 --- a/conformance/authority_adapter.py +++ b/conformance/authority_adapter.py @@ -170,7 +170,13 @@ def _observed_ledger(authority: SQLiteLocalAuthority) -> dict[str, Any] | None: return { key: value for key, value in ledger.items() - if key not in {"native_checkpoint_bytes", "effect_claim_history", "native_effect_roots"} + if key + not in { + "native_checkpoint_bytes", + "effect_claim_history", + "native_effect_roots", + "native_effect_work", + } } diff --git a/src/determa/state/authority.py b/src/determa/state/authority.py index 807efe2..0147357 100644 --- a/src/determa/state/authority.py +++ b/src/determa/state/authority.py @@ -965,8 +965,22 @@ def _perform( } # An installed effect journal is a participant in this claim, # not an eventually updated mirror of the authority receipt. - native_owned = arguments["root_instance_id"] in ledger.get( - "native_effect_roots", [] + native_binding = next( + ( + item + for item in ledger.get("native_effect_work", []) + if item["work_identity"] == arguments["work_identity"] + ), + None, + ) + if native_binding is not None and any( + native_binding[key] != arguments[key] + for key in ("root_instance_id", "work_kind", "operation_token") + ): + return _compact(_result(request, ledger, "invalid_host_request")) + native_owned = ( + arguments["root_instance_id"] in ledger.get("native_effect_roots", []) + or native_binding is not None ) native_update = None if connection.execute( diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 9d8f341..2cc2620 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -375,6 +375,15 @@ def _validate_authority_pair(ledger: Mapping[str, Any], document: Mapping[str, A ): raise EffectError("unauthorized_scope") for record in journal["effect_records"]: + binding = { + "root_instance_id": journal["root_instance_id"], + "work_kind": "effect", + "work_identity": record["effect_id"], + "operation_token": record["operation_token"], + "participant": "native_effects", + } + if binding not in ledger.get("native_effect_work", []): + raise EffectError("unauthorized_scope") issued = next( ( entry @@ -600,6 +609,21 @@ def _mirror_authority( native_roots.append(root) records = document["journal"]["effect_records"] for record in records: + binding = { + "root_instance_id": root, + "work_kind": "effect", + "work_identity": record["effect_id"], + "operation_token": record["operation_token"], + "participant": "native_effects", + } + bindings = ledger.setdefault("native_effect_work", []) + existing = next( + (item for item in bindings if item["work_identity"] == record["effect_id"]), None + ) + if existing is not None and existing != binding: + raise EffectError("stale_attempt_fence") + if existing is None: + bindings.append(binding) entry = next( ( item diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 78de134..11ec606 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -422,3 +422,16 @@ def test_seed_preserves_historical_claim_without_reviving_it(tmp_path): assert new_claim["attempt_fence"] == "2" assert authority.inspect(scope)["active_claims"] == [new_claim] assert claim in authority.inspect(scope)["effect_claim_history"] + + +def test_missing_native_work_cannot_be_rebound_to_another_authorized_root(tmp_path): + authority, host, scope, root, record = authority_effect_fixture(tmp_path) + ledger = authority.inspect(scope) + ledger["roots"].append("other-authorized-root") + with authority._connect() as connection: + connection.execute("UPDATE determa_scope_authority SET ledger=?", (json.dumps(ledger),)) + connection.execute("DELETE FROM determa_committed_effects") + connection.commit() + request, invocation = authority_claim_request(authority, scope, "other-authorized-root", record) + assert json.loads(authority.perform(request, invocation))["status"] == "rejected" + assert authority.inspect(scope) == ledger From e45dc9d9e27b3eeafe4ad0e11e4119ea282fc91c Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 06:06:18 +0000 Subject: [PATCH 11/30] Pin current conformance and require composed native effects CI gate --- .github/workflows/test.yml | 10 +++++++--- AGENTS.md | 4 ++-- conformance/pins.py | 2 +- conformance/test_conformance.py | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b199eaf..2e2297c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,7 +53,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: ce6c94ee1929597689403080d1790e2f48f8a5fb + ref: 7f09321fb483a22eb677a4342f8d9537a7a18e82 path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -88,7 +88,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: ce6c94ee1929597689403080d1790e2f48f8a5fb + ref: 7f09321fb483a22eb677a4342f8d9537a7a18e82 path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -126,7 +126,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: ce6c94ee1929597689403080d1790e2f48f8a5fb + ref: 7f09321fb483a22eb677a4342f8d9537a7a18e82 path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -151,6 +151,10 @@ jobs: PYTHONPATH=src python "$DETERMA_CONFORMANCE_DIR/scripts/run_host_authority_profile.py" \ --spec-root "$DETERMA_SPEC_DIR" \ --adapter python conformance/authority_adapter.py + PYTHONPATH=src python "$DETERMA_CONFORMANCE_DIR/scripts/run_committed_native_effects_profile.py" \ + --spec-root "$DETERMA_SPEC_DIR" \ + --adapter "python conformance/effects_adapter.py" \ + --authority-adapter "python conformance/authority_adapter.py --worker" postgresql: runs-on: ubuntu-24.04 diff --git a/AGENTS.md b/AGENTS.md index d492bc8..b126b5d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,10 +12,10 @@ The implementation is conformant only when it passes the language-neutral suite. The implementation uses these immutable inputs: - specification: `86bb88dd21cb1f799eefe5020b6e49dabf6e7225`; -- conformance: `ce6c94ee1929597689403080d1790e2f48f8a5fb` (99 format-1 +- conformance: `7f09321fb483a22eb677a4342f8d9537a7a18e82` (99 format-1 core cases, 162 version-1 vectors, 142 durable-host vectors, 49 core inspection vectors, 47 extension negotiation vectors, 7 optional native-provider vectors, - 30 runtime-provider vectors, and 476 JSON artifacts). + 30 runtime-provider vectors, and 484 JSON artifacts). The package metadata is `0.3.0`. diff --git a/conformance/pins.py b/conformance/pins.py index 1feed69..ea356eb 100644 --- a/conformance/pins.py +++ b/conformance/pins.py @@ -4,7 +4,7 @@ from pathlib import Path -CONFORMANCE_COMMIT = "ce6c94ee1929597689403080d1790e2f48f8a5fb" +CONFORMANCE_COMMIT = "7f09321fb483a22eb677a4342f8d9537a7a18e82" SPEC_COMMIT = "86bb88dd21cb1f799eefe5020b6e49dabf6e7225" ROOT = Path(__file__).resolve().parent.parent diff --git a/conformance/test_conformance.py b/conformance/test_conformance.py index 296f91e..a1f0006 100644 --- a/conformance/test_conformance.py +++ b/conformance/test_conformance.py @@ -147,7 +147,7 @@ def test_suite_present() -> None: assert len(core_cases()) == 99 assert len(version1_vectors()) == 162 assert len(durable_host_vectors()) == 142 - assert len(_manifest_artifacts()) == 476 + assert len(_manifest_artifacts()) == 484 @pytest.mark.parametrize("stored", [b"mutated", None]) From 01c1ae69491b0651d796cf60f41d80b0dd7f0537 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 06:08:02 +0000 Subject: [PATCH 12/30] Use worker authority wrapper for the composed conformance runner --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e2297c..880b0f2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -154,7 +154,7 @@ jobs: PYTHONPATH=src python "$DETERMA_CONFORMANCE_DIR/scripts/run_committed_native_effects_profile.py" \ --spec-root "$DETERMA_SPEC_DIR" \ --adapter "python conformance/effects_adapter.py" \ - --authority-adapter "python conformance/authority_adapter.py --worker" + --authority-adapter "python conformance/worker_authority_adapter.py" postgresql: runs-on: ubuntu-24.04 From fc093f0ae20140dada4355015088c237270da24c Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 06:08:28 +0000 Subject: [PATCH 13/30] Include repository adapter package in composed runner imports --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 880b0f2..94dc77a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -151,7 +151,7 @@ jobs: PYTHONPATH=src python "$DETERMA_CONFORMANCE_DIR/scripts/run_host_authority_profile.py" \ --spec-root "$DETERMA_SPEC_DIR" \ --adapter python conformance/authority_adapter.py - PYTHONPATH=src python "$DETERMA_CONFORMANCE_DIR/scripts/run_committed_native_effects_profile.py" \ + PYTHONPATH=src:. python "$DETERMA_CONFORMANCE_DIR/scripts/run_committed_native_effects_profile.py" \ --spec-root "$DETERMA_SPEC_DIR" \ --adapter "python conformance/effects_adapter.py" \ --authority-adapter "python conformance/worker_authority_adapter.py" From 6b33b7c929b8425ead2ce68618350ce610014e03 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 06:13:42 +0000 Subject: [PATCH 14/30] Validate committed effect checkpoints and retain terminal intent dispatch --- conformance/test_conformance.py | 3 ++- src/determa/state/effects.py | 6 +++++- tests/test_committed_effects.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/conformance/test_conformance.py b/conformance/test_conformance.py index a1f0006..ddbed0b 100644 --- a/conformance/test_conformance.py +++ b/conformance/test_conformance.py @@ -110,7 +110,8 @@ def _validate_manifest_artifact(path: Path, artifact: dict) -> None: registry = _schema_registry() for reference_path in sorted(schema_path.parent.glob("*.schema.json")): reference = json.loads(reference_path.read_text(encoding="utf-8")) - registry = registry.with_resource(reference["$id"], Resource.from_contents(reference)) + if "$id" in reference: + registry = registry.with_resource(reference["$id"], Resource.from_contents(reference)) spec_root = _spec_root() if spec_root is not None: for reference_path in sorted((spec_root / "schema").glob("*.schema.json")): diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 2cc2620..e1d8d73 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -756,6 +756,7 @@ def _transact( before = canonical_bytes(document) value = change(document) validate_journal(document["checkpoint"], document["journal"]) + restore_execution_checkpoint_v1(document["checkpoint"], self.resolver) if ( ledger is not None and authority_mutation is not None @@ -1031,7 +1032,10 @@ def call(document: dict[str, Any]) -> Mapping[str, Any]: raise EffectError("unauthorized_scope") intent = next( item["intent"] - for item in document["checkpoint"]["pending_outbox_intents"] + for item in ( + document["checkpoint"]["pending_outbox_intents"] + + document["checkpoint"]["terminal_outbox_records"] + ) if item["intent"]["effect_id"] == effect_id ) metadata = { diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 11ec606..0782dea 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -435,3 +435,32 @@ def test_missing_native_work_cannot_be_rebound_to_another_authorized_root(tmp_pa request, invocation = authority_claim_request(authority, scope, "other-authorized-root", record) assert json.loads(authority.perform(request, invocation))["status"] == "rejected" assert authority.inspect(scope) == ledger + + +def test_invalid_terminal_outcome_cannot_corrupt_persisted_checkpoint(tmp_path): + from determa.state.wire import ArtifactError + + host, root, request, _ = host_fixture(tmp_path) + before = host.snapshot(root) + with pytest.raises(ArtifactError): + host.terminalize_outbox(root, request["effect_id"], {"status": "bogus"}) + assert host.snapshot(root) == before + + +def test_confirmed_outbox_does_not_remove_the_business_invocation_payload(tmp_path): + host, root, request, context = host_fixture(tmp_path) + record = host.snapshot(root)["journal"]["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + calls = [] + host.handler = lambda *arguments: calls.append(arguments) or {"invoked": True} + host.terminalize_outbox(root, request["effect_id"], {"status": "confirmed"}) + assert host.dispatch(root, request["effect_id"], credential="test-credential", **context) == { + "invoked": True + } + assert len(calls) == 1 + assert ( + calls[0][0] + == host.snapshot(root)["checkpoint"]["terminal_outbox_records"][0]["intent"]["payload"] + ) From 701ca50572c35a90f2454000fe594bbb34c8878d Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 06:16:50 +0000 Subject: [PATCH 15/30] Recover each outstanding effect in a separate journal transaction --- src/determa/state/effects.py | 17 +++++---- tests/test_committed_effects.py | 67 +++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 8 deletions(-) diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index e1d8d73..3e04ef4 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -1310,8 +1310,14 @@ def _admit(self, document: dict[str, Any], record: dict[str, Any]) -> None: ) def recover(self, root: str) -> dict[str, Any]: - def change(document: dict[str, Any]) -> dict[str, Any]: - for record in document["journal"]["effect_records"]: + # Each recovered effect has its own atomic journal transition. A crash + # between records leaves the remaining records available to recover. + snapshot = self.snapshot(root) + for selected in snapshot["journal"]["effect_records"]: + effect_id = selected["effect_id"] + + def change(document: dict[str, Any], effect_id: str = effect_id) -> None: + record = _record(document["journal"], effect_id) if record["invocation_state"] == "outcome_recorded": self._admit(document, record) elif record["invocation_state"] == "leased": @@ -1337,19 +1343,14 @@ def change(document: dict[str, Any]) -> dict[str, Any]: record["invocation_state"] = "ambiguous" document["claims"][record["effect_id"]]["state"] = "revoked" _bump(document["journal"]) - return copy.deepcopy(document) - snapshot = self.snapshot(root) - return ( self._transact( root, change, expected_epoch=self.route.get("authority_epoch"), authority_mutation=self._mirror_authority, ) - if snapshot["journal"]["effect_records"] - else snapshot - ) + return self.snapshot(root) def cancel(self, root: str, request: Mapping[str, Any]) -> dict[str, Any]: def change(document: dict[str, Any]) -> dict[str, Any]: diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 0782dea..b8645dc 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -464,3 +464,70 @@ def test_confirmed_outbox_does_not_remove_the_business_invocation_payload(tmp_pa calls[0][0] == host.snapshot(root)["checkpoint"]["terminal_outbox_records"][0]["intent"]["payload"] ) + + +def test_multi_effect_recovery_commits_one_journal_revision_per_effect(tmp_path): + from determa.state.checkpoint_v1 import admit_checkpoint_v1, step_checkpoint_v1 + from determa.state.host import outbox_intent_digest + from determa.state.queueing import _entry_digest + + original, root, _, _ = host_fixture(tmp_path) + checkpoint = original.snapshot(root)["checkpoint"] + entry = read("accepted-checkpoint.json")["root_record"]["aggregate_state"]["runtimes"][0][ + "ready_mailbox" + ][0] + envelope = copy.deepcopy(entry["envelope"]) + envelope["event_id"] = "invoke-second" + envelope["cause_id"] = "invoke-second" + delivery = { + "delivery_mode": entry["delivery_mode"], + "envelope": envelope, + "envelope_digest": _entry_digest(root, entry["delivery_mode"], envelope), + } + admitted = admit_checkpoint_v1( + checkpoint, + [delivery], + original.resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + checkpoint = step_checkpoint_v1( + admitted, + checkpoint["root_record"]["aggregate_state"]["root_runtime_id"], + original.resolver, + expected_revision=admitted["revision"], + expected_checkpoint_digest=admitted["execution_checkpoint_digest"], + ) + journal = read("data/unclaimed-journal.json") + template = journal["effect_records"][0] + journal["effect_records"] = [] + for item in checkpoint["pending_outbox_intents"]: + record = copy.deepcopy(template) + record["effect_id"] = item["intent"]["effect_id"] + record["intent_digest"] = outbox_intent_digest(root, item["intent"]) + journal["effect_records"].append(record) + journal["effect_records"].sort(key=lambda record: record["effect_id"].encode()) + journal["checkpoint_revision"] = checkpoint["revision"] + journal["checkpoint_digest"] = checkpoint["execution_checkpoint_digest"] + journal = seal_journal(journal) + host = SQLiteCommittedEffectHost(tmp_path / "two.sqlite", original.resolver, {}, lambda *_: {}) + host.setup_schema() + host.seed(checkpoint, journal) + for record in journal["effect_records"]: + host.claim(root, record["effect_id"], "worker", "0", expires_at="10", trusted_now="0") + initial_revision = int(host.snapshot(root)["journal"]["journal_revision"]) + committed_revisions = [] + transact = host._transact + + def observe_transaction(*args, **kwargs): + result = transact(*args, **kwargs) + committed_revisions.append(int(host.snapshot(root)["journal"]["journal_revision"])) + return result + + host._transact = observe_transaction + recovered = host.recover(root) + assert len(recovered["journal"]["effect_records"]) == 2 + assert {record["invocation_state"] for record in recovered["journal"]["effect_records"]} == { + "ambiguous" + } + assert committed_revisions == [initial_revision + 1, initial_revision + 2] From f6e2f26662f5b72d6623687b0fe49385b1d1d6c0 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 06:19:53 +0000 Subject: [PATCH 16/30] Recheck the trusted worker clock after native handler execution --- conformance/effects_adapter.py | 1 + src/determa/state/effects.py | 22 ++++++++++++++++++++-- tests/test_committed_effects.py | 20 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/conformance/effects_adapter.py b/conformance/effects_adapter.py index 7bdb727..8ba1b68 100644 --- a/conformance/effects_adapter.py +++ b/conformance/effects_adapter.py @@ -318,6 +318,7 @@ def observe_core(kind: str, event_id: str, target: dict[str, Any]) -> None: handler, authority_scope=_AUTHORITY_SCOPE, core_observer=observe_core, + trusted_clock=lambda: payload["auth_context"]["trusted_host_now"], ) # Install the vector's pre-existing authority facts as trusted test setup. # The production seed path may restore these claims, but cannot issue them. diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 3e04ef4..16b0352 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -474,6 +474,7 @@ def __init__( *, authority_scope: str | None = None, core_observer: Callable[[str, str, Mapping[str, Any]], None] | None = None, + trusted_clock: Callable[[], str] | None = None, ) -> None: self.path = str(Path(path).resolve()) self.resolver = resolver @@ -481,6 +482,7 @@ def __init__( self.handler = handler self.authority_scope = authority_scope self.core_observer = core_observer + self.trusted_clock = trusted_clock def _connect(self) -> sqlite3.Connection: connection = sqlite3.connect(self.path, isolation_level=None, timeout=30) @@ -1008,6 +1010,9 @@ def dispatch( authorized: bool = True, ) -> Mapping[str, Any]: """Run an installed native handler after durable intent and live guard proof.""" + clock = self.trusted_clock + if clock is None: + raise EffectError("stale_attempt_fence") def call(document: dict[str, Any]) -> Mapping[str, Any]: record = _record(document["journal"], effect_id) @@ -1021,7 +1026,7 @@ def call(document: dict[str, Any]) -> Mapping[str, Any]: principal, scope, epoch, - trusted_now, + clock(), ) if ( not authorized @@ -1045,11 +1050,24 @@ def call(document: dict[str, Any]) -> Mapping[str, Any]: "destination_binding_digest": record["destination_binding_digest"], "credential": credential, } - return self.handler( + result = self.handler( copy.deepcopy(intent["payload"]), metadata, {"attempt_fence": record["attempt_fence"]}, ) + # External acceptance cannot extend an expired worker's mutation + # rights. A refusal leaves its leased journal for reconciliation. + self._authorize( + document, + effect_id, + record["operation_token"], + record["attempt_fence"], + principal, + scope, + epoch, + clock(), + ) + return result return self._transact( root, call, expected_epoch=epoch, worker_guard=(effect_id, principal, None, trusted_now) diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index b8645dc..e5ca75d 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -455,6 +455,7 @@ def test_confirmed_outbox_does_not_remove_the_business_invocation_payload(tmp_pa ) calls = [] host.handler = lambda *arguments: calls.append(arguments) or {"invoked": True} + host.trusted_clock = lambda: "0" host.terminalize_outbox(root, request["effect_id"], {"status": "confirmed"}) assert host.dispatch(root, request["effect_id"], credential="test-credential", **context) == { "invoked": True @@ -531,3 +532,22 @@ def observe_transaction(*args, **kwargs): "ambiguous" } assert committed_revisions == [initial_revision + 1, initial_revision + 2] + + +def test_dispatch_crossing_lease_expiry_preserves_unresolved_work_for_recovery(tmp_path): + host, root, request, context = host_fixture(tmp_path) + record = host.snapshot(root)["journal"]["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + expiry = read("data/active-claim.json")["expires_at"] + readings = iter(["0", expiry]) + host.trusted_clock = lambda: next(readings) + calls = [] + host.handler = lambda *args: calls.append(args) or {"accepted": True} + before = host.snapshot(root) + with pytest.raises(EffectError, match="stale_attempt_fence"): + host.dispatch(root, request["effect_id"], credential="test-credential", **context) + assert len(calls) == 1 + assert host.snapshot(root) == before + assert host.recover(root)["journal"]["effect_records"][0]["invocation_state"] == "ambiguous" From 5c67aaaa74d8e8349f024e8835b8694c59153c07 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 06:24:08 +0000 Subject: [PATCH 17/30] Checkpoint verified native-handler handles; adapter wiring remains pending --- src/determa/state/effects.py | 77 +++++++++++++++++++++- tests/test_committed_effects.py | 113 +++++++++++++++++++++++++++++++- 2 files changed, 186 insertions(+), 4 deletions(-) diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 16b0352..9d0c927 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -23,6 +23,7 @@ step_checkpoint_v1, ) from .errors import ArtifactError +from .extensions import ConfiguredExtension, ExtensionError, ExtensionRegistry from .host import outbox_intent_digest from .wire import ( ArtifactResolver, @@ -456,6 +457,56 @@ def _issue_effect_claim( return copy.deepcopy(claim) +class VerifiedNativeHandler: + """An exact native-handler instance installed through the public registry. + + The host-owned registry verifier must verify its executing source and + dependency closure. This handle never promotes provider claims into proof. + """ + + def __init__(self, registry: ExtensionRegistry, configured: ConfiguredExtension) -> None: + self._registry = registry + self._configured = configured + descriptor, _provider, _instance, _evaluator = registry._bound(configured) + if descriptor["category"] != "native_handler": + raise EffectError("host_capability_mismatch") + self._reference = copy.deepcopy(descriptor["provider_reference"]) + self._configuration = copy.deepcopy(configured._configuration) + destination = self._configuration.get("destination_binding_digest") + if type(destination) is not str or not re.fullmatch(r"sha256:[0-9a-f]{64}", destination): + raise EffectError("host_capability_mismatch") + + def verify(self, reference: Mapping[str, Any], destination: str) -> tuple[Any, Any]: + from .runtime_providers import _bound_provider_method + + try: + descriptor, provider, instance, _evaluator = self._registry._bound(self._configured) + if ( + descriptor["category"] != "native_handler" + or descriptor["provider_reference"] != self._reference + or reference != self._reference + or self._configured._configuration != self._configuration + or destination != self._configuration["destination_binding_digest"] + or not isinstance(instance, Mapping) + or instance.get("destination_binding_digest") != destination + or self._registry.health(self._configured) != "healthy" + ): + raise EffectError("host_capability_mismatch") + return _bound_provider_method(provider, "invoke"), instance + except (ExtensionError, ValueError, TypeError, KeyError) as error: + raise EffectError("host_capability_mismatch") from error + + def invoke( + self, + reference: Mapping[str, Any], + payload: Any, + metadata: Mapping[str, Any], + attempt: Mapping[str, Any], + ) -> Mapping[str, Any]: + method, instance = self.verify(reference, metadata["destination_binding_digest"]) + return cast(Mapping[str, Any], method(instance, payload, metadata, attempt)) + + class SQLiteCommittedEffectHost: """One local host with checkpoint, journal, claims and responses in one SQLite row. @@ -470,7 +521,7 @@ def __init__( path: str | Path, resolver: ArtifactResolver, route: Mapping[str, Any], - handler: Callable[..., Mapping[str, Any]], + handler: VerifiedNativeHandler | Callable[..., Mapping[str, Any]], *, authority_scope: str | None = None, core_observer: Callable[[str, str, Mapping[str, Any]], None] | None = None, @@ -480,10 +531,22 @@ def __init__( self.resolver = resolver self.route = copy.deepcopy(dict(route)) self.handler = handler + self._installed_handler = handler self.authority_scope = authority_scope self.core_observer = core_observer self.trusted_clock = trusted_clock + def _verified_handler( + self, reference: Mapping[str, Any], destination: str + ) -> VerifiedNativeHandler: + if ( + type(self.handler) is not VerifiedNativeHandler + or self.handler is not self._installed_handler + ): + raise EffectError("host_capability_mismatch") + self.handler.verify(reference, destination) + return self.handler + def _connect(self) -> sqlite3.Connection: connection = sqlite3.connect(self.path, isolation_level=None, timeout=30) connection.execute("PRAGMA journal_mode=WAL") @@ -806,6 +869,9 @@ def change(document: dict[str, Any]) -> dict[str, Any]: return reconstructed if route_generation != self.route["generation"]: raise EffectError("scope_generation_conflict") + self._verified_handler( + self.route["handler_reference"], self.route["destination_binding_digest"] + ) if after_route_resolved is not None: after_route_resolved(self.route) checkpoint, journal = document["checkpoint"], document["journal"] @@ -880,6 +946,9 @@ def change(document: dict[str, Any]) -> dict[str, Any]: before_commit(candidate, journal) if route_generation != self.route["generation"]: raise EffectError("scope_generation_conflict") + self._verified_handler( + self.route["handler_reference"], self.route["destination_binding_digest"] + ) return response return self._transact( @@ -1050,7 +1119,11 @@ def call(document: dict[str, Any]) -> Mapping[str, Any]: "destination_binding_digest": record["destination_binding_digest"], "credential": credential, } - result = self.handler( + handler = self._verified_handler( + record["handler_reference"], record["destination_binding_digest"] + ) + result = handler.invoke( + record["handler_reference"], copy.deepcopy(intent["payload"]), metadata, {"attempt_fence": record["attempt_fence"]}, diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index e5ca75d..c76cea8 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -21,6 +21,74 @@ def read(name): return json.loads((CASE / name).read_text()) +class NativeTestProvider: + def __init__(self, callback): + self.callback = callback + + def validate_configuration(self, configuration): + return copy.deepcopy(configuration) + + def capabilities(self, instance): + return [] + + def health(self, instance): + return "healthy" + + def invoke(self, instance, payload, metadata, attempt): + return self.callback(payload, metadata, attempt) + + +def installed_test_handler(host, callback): + from determa.state.effects import VerifiedNativeHandler + from determa.state.extensions import ExtensionRegistry + + provider = NativeTestProvider(callback) + + def factory(): + return provider + + callback_code = callback.__code__ + factory_code = factory.__code__ + methods = { + name: member.__code__ + for name, member in vars(NativeTestProvider).items() + if callable(member) + } + descriptor = { + "category": "native_handler", + "provider_reference": host.route["handler_reference"], + "interface_version": 1, + "supported_capabilities": [], + } + + def verify(source, actual): + return ( + actual == descriptor + and (source is factory or source is provider) + and factory.__code__ is factory_code + and type(provider) is NativeTestProvider + and provider.callback is callback + and callback.__code__ is callback_code + and all( + getattr(NativeTestProvider, name).__code__ is code for name, code in methods.items() + ) + ) + + registry = ExtensionRegistry(source_verifier=verify) + registry.register(descriptor, factory) + configured = registry.validate_configuration( + descriptor, + { + "instance_id": "test-handler", + "destination_binding_digest": host.route["destination_binding_digest"], + }, + ) + handler = VerifiedNativeHandler(registry, configured) + return SQLiteCommittedEffectHost( + host.path, host.resolver, host.route, handler, trusted_clock=host.trusted_clock + ) + + def host_fixture(tmp_path, observer=None): bundle = load_bundle((CASE / "machine.yaml").read_text()) resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) @@ -454,7 +522,9 @@ def test_confirmed_outbox_does_not_remove_the_business_invocation_payload(tmp_pa {key: record[key] for key in ("handler_reference", "destination_binding_digest")} ) calls = [] - host.handler = lambda *arguments: calls.append(arguments) or {"invoked": True} + host = installed_test_handler( + host, lambda *arguments: calls.append(arguments) or {"invoked": True} + ) host.trusted_clock = lambda: "0" host.terminalize_outbox(root, request["effect_id"], {"status": "confirmed"}) assert host.dispatch(root, request["effect_id"], credential="test-credential", **context) == { @@ -544,10 +614,49 @@ def test_dispatch_crossing_lease_expiry_preserves_unresolved_work_for_recovery(t readings = iter(["0", expiry]) host.trusted_clock = lambda: next(readings) calls = [] - host.handler = lambda *args: calls.append(args) or {"accepted": True} + host = installed_test_handler(host, lambda *args: calls.append(args) or {"accepted": True}) before = host.snapshot(root) with pytest.raises(EffectError, match="stale_attempt_fence"): host.dispatch(root, request["effect_id"], credential="test-credential", **context) assert len(calls) == 1 assert host.snapshot(root) == before assert host.recover(root)["journal"]["effect_records"][0]["invocation_state"] == "ambiguous" + + +@pytest.mark.parametrize( + "replacement", ["raw_callback", "callback_code", "method", "configuration", "instance"] +) +def test_native_handler_binding_rejects_changed_execution_or_destination( + tmp_path, monkeypatch, replacement +): + host, root, request, context = host_fixture(tmp_path) + record = host.snapshot(root)["journal"]["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + calls = [] + + def callback(*args): + return calls.append(args) or {} + + host = installed_test_handler(host, callback) + host.trusted_clock = lambda: "0" + if replacement == "raw_callback": + host.handler = callback + elif replacement == "callback_code": + + def other(*args): + return calls or {"changed": True} + + monkeypatch.setattr(callback, "__code__", other.__code__) + elif replacement == "method": + monkeypatch.setattr(NativeTestProvider, "invoke", lambda *args: {}) + elif replacement == "configuration": + host.handler._configured._configuration["destination_binding_digest"] = "sha256:" + "0" * 64 + else: + host.handler._configured._instance["destination_binding_digest"] = "sha256:" + "0" * 64 + before = host.snapshot(root) + with pytest.raises(EffectError, match="host_capability_mismatch"): + host.dispatch(root, request["effect_id"], credential="test-credential", **context) + assert calls == [] + assert host.snapshot(root) == before From 5bf335c996fb65095aa2780927acdb6df3c7cf4b Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 06:28:32 +0000 Subject: [PATCH 18/30] Bind native handler adapter execution through verified public registration --- conformance/effects_adapter.py | 230 ++++++++++++++++++++++++++++++-- src/determa/state/effects.py | 2 + tests/test_committed_effects.py | 13 ++ 3 files changed, 233 insertions(+), 12 deletions(-) diff --git a/conformance/effects_adapter.py b/conformance/effects_adapter.py index 8ba1b68..5397cd8 100644 --- a/conformance/effects_adapter.py +++ b/conformance/effects_adapter.py @@ -3,8 +3,11 @@ from __future__ import annotations +import ast import base64 +import builtins import copy +import dataclasses import hashlib import importlib.util import json @@ -13,12 +16,14 @@ import subprocess import sys import time +import types import uuid from pathlib import Path from typing import Any from conformance.authority_adapter import _PATH from determa.state import load_bundle +from determa.state._platform_bindings import PLATFORM_BINDINGS from determa.state.authority import ( AuthoritySQLiteExecutionStore, SQLiteLocalAuthority, @@ -28,9 +33,10 @@ from determa.state.effects import ( EffectError, SQLiteCommittedEffectHost, + VerifiedNativeHandler, _result_response, ) -from determa.state.extensions import bundled_extension_registry +from determa.state.extensions import ExtensionRegistry, bundled_extension_registry from determa.state.wire import ( MemoryArtifactResolver, canonical_bytes, @@ -258,6 +264,146 @@ def _loaded_handler() -> Any: return module +def _verified_fixture_handler(module: Any) -> bool: + """Check the actual fixture function and its narrow generated dataclass closure. + + Only this allowlisted two-field frozen class is supported. Reference generation + uses anchored standard-library machinery, never a provider-supplied decorator. + """ + path = _CASE / _HANDLER_SOURCE + source = path.read_bytes() + tree = ast.parse(source) + declaration = next((node for node in tree.body if isinstance(node, ast.ClassDef)), None) + if ( + declaration is None + or declaration.name != "NativeReply" + or declaration.bases + or declaration.keywords + or len(declaration.decorator_list) != 1 + or ast.dump(declaration.decorator_list[0]) + != ast.dump(ast.parse("dataclass(frozen=True)", mode="eval").body) + or len(declaration.body) != 2 + ): + return False + for node, name, kind in zip( + declaration.body, ("reference", "accepted"), ("str", "bool"), strict=True + ): + if not ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == name + and isinstance(node.annotation, ast.Name) + and node.annotation.id == kind + and node.value is None + ): + return False + if ( + not PLATFORM_BINDINGS.matches(dataclasses, "dataclass") + or not PLATFORM_BINDINGS.matches(dataclasses, "make_dataclass") + or not PLATFORM_BINDINGS.matches(dataclasses, "FrozenInstanceError") + or module.dataclass is not dataclasses.dataclass + ): + return False + expected = dataclasses.make_dataclass( + "NativeReply", + [("reference", str), ("accepted", bool)], + namespace={"__module__": module.__name__}, + frozen=True, + ) + expected.__module__ = module.__name__ + actual = module.NativeReply + if ( + type(actual) is not type + or actual.__bases__ != (object,) + or vars(actual).keys() != vars(expected).keys() + ): + return False + seen = set() + + def equal(left: Any, right: Any) -> bool: + if left is actual or right is expected: + return left is actual and right is expected + if type(left) is not type(right): + return False + if isinstance(left, types.FunctionType): + pair = (id(left), id(right)) + if pair in seen: + return True + seen.add(pair) + return ( + left.__code__ == right.__code__ + and all( + equal( + left.__globals__.get(name, vars(builtins).get(name)), + right.__globals__.get(name, vars(builtins).get(name)), + ) + for name in right.__code__.co_names + if name in left.__globals__ + or name in right.__globals__ + or name in vars(builtins) + ) + and equal(left.__defaults__, right.__defaults__) + and equal(left.__kwdefaults__, right.__kwdefaults__) + and equal(vars(left), vars(right)) + and equal( + tuple(cell.cell_contents for cell in left.__closure__ or ()), + tuple(cell.cell_contents for cell in right.__closure__ or ()), + ) + ) + if isinstance(left, (tuple, list)): + return len(left) == len(right) and all( + equal(a, b) for a, b in zip(left, right, strict=True) + ) + if type(left) is dict: + return left.keys() == right.keys() and all(equal(left[key], right[key]) for key in left) + if isinstance(left, (types.GetSetDescriptorType, types.MemberDescriptorType)): + return ( + left.__name__ == right.__name__ + and left.__objclass__ is actual + and right.__objclass__ is expected + ) + if type(left) in (dataclasses.Field, type(expected.__dataclass_params__)): + return all( + equal(getattr(left, name), getattr(right, name)) + for name in type(left).__slots__ + if name != "__weakref__" + ) + return left == right + + if not all(equal(member, vars(expected)[name]) for name, member in vars(actual).items()): + return False + code = next( + item + for item in compile(source, str(path), "exec", dont_inherit=True).co_consts + if isinstance(item, types.CodeType) and item.co_name == "invoke" + ) + return ( + type(module.invoke) is types.FunctionType + and module.invoke.__code__ == code + and module.invoke.__globals__ is vars(module) + and module.invoke.__closure__ is None + and module.invoke.__defaults__ is None + and module.invoke.__kwdefaults__ is None + ) + + +class _FixtureNativeProvider: + def __init__(self, invocation: Any) -> None: + self.invocation = invocation + + def validate_configuration(self, configuration: dict[str, Any]) -> dict[str, Any]: + return copy.deepcopy(configuration) + + def capabilities(self, instance: Any) -> list[str]: + return [] + + def health(self, instance: Any) -> str: + return "healthy" + + def invoke(self, instance: Any, payload: Any, metadata: Any, attempt: Any) -> Any: + return self.invocation(payload, metadata, attempt) + + def _run_operation(payload: dict[str, Any], control: Path | None = None) -> dict[str, Any]: checkpoint, journal = payload["checkpoint_before"], payload["journal_before"] authority, authority_report, _installation = _reset_authority() @@ -278,29 +424,33 @@ def _run_operation(payload: dict[str, Any], control: Path | None = None) -> dict loaded_handler_source: dict[str, str] = {} destination_call_evidence: dict[str, Any] | None = None + active_attempt: dict[str, Any] = {} + active_metadata: dict[str, Any] = {} + class CallLog: def call(self, scope: str, effect_id: str, portable_payload: Any) -> tuple[str, bool]: nonlocal destination_call_evidence destination_call_evidence = { "idempotency_key": [scope, effect_id], - "destination_binding_digest": route["destination_binding_digest"], - "attempt_fence": journal["effect_records"][0]["attempt_fence"], + "destination_binding_digest": active_metadata["destination_binding_digest"], + "attempt_fence": active_attempt["attempt_fence"], } return _destination_call(scope, effect_id, portable_payload) def handler( typed_payload: Any, metadata: dict[str, Any], attempt: dict[str, Any] ) -> dict[str, Any]: - record = journal["effect_records"][0] + active_attempt.update(attempt) + active_metadata.update(metadata) provider_calls.append( { - "effect_id": record["effect_id"], - "handler_reference": record["handler_reference"], - "destination_binding_digest": record["destination_binding_digest"], - "route_configuration_generation": record["route_configuration_generation"], - "attempt_fence": record["attempt_fence"], - "scope_identity": _AUTHORITY_SCOPE, - "credential_generation": configuration["credential_generation"], + "effect_id": metadata["effect_id"], + "handler_reference": descriptor["provider_reference"], + "destination_binding_digest": metadata["destination_binding_digest"], + "route_configuration_generation": metadata["route_configuration_generation"], + "attempt_fence": attempt["attempt_fence"], + "scope_identity": metadata["scope_identity"], + "credential_generation": metadata["credential"], } ) loaded_handler_source[_HANDLER_SOURCE] = _sha((_CASE / _HANDLER_SOURCE).read_bytes()) @@ -311,11 +461,67 @@ def observe_core(kind: str, event_id: str, target: dict[str, Any]) -> None: {"operation_kind": kind, "event_id": event_id, "target": copy.deepcopy(target)} ) + provider = _FixtureNativeProvider(handler) + + def factory() -> _FixtureNativeProvider: + return provider + + factory_code, handler_code = factory.__code__, handler.__code__ + installed_invoke, installed_reply = handler_module.invoke, handler_module.NativeReply + installed_members = { + name: member + for name, member in vars(installed_reply).items() + if callable(member) or hasattr(type(member), "__get__") + } + provider_methods = { + name: member.__code__ + for name, member in vars(_FixtureNativeProvider).items() + if isinstance(member, types.FunctionType) + } + descriptor = { + "category": "native_handler", + "provider_reference": route["handler_reference"], + "interface_version": 1, + "supported_capabilities": [], + } + + def verify_native(source: Any, selected: Any) -> bool: + return ( + selected == descriptor + and (source is factory or source is provider) + and factory.__code__ is factory_code + and type(provider) is _FixtureNativeProvider + and provider.invocation is handler + and handler.__code__ is handler_code + and all( + getattr(_FixtureNativeProvider, name).__code__ is code + for name, code in provider_methods.items() + ) + and _sha(_handler_closure()) == selected["provider_reference"]["content_digest"] + and handler_module.invoke is installed_invoke + and handler_module.NativeReply is installed_reply + and all( + vars(installed_reply).get(name) is member + for name, member in installed_members.items() + ) + and _verified_fixture_handler(handler_module) + ) + + handler_registry = ExtensionRegistry(source_verifier=verify_native) + handler_registry.register(descriptor, factory) + configured_handler = handler_registry.validate_configuration( + descriptor, + { + "instance_id": "committed-effect-handler", + "destination_binding_digest": route["destination_binding_digest"], + }, + ) + installed_handler = VerifiedNativeHandler(handler_registry, configured_handler) host = SQLiteCommittedEffectHost( _PATH, resolver, route, - handler, + installed_handler, authority_scope=_AUTHORITY_SCOPE, core_observer=observe_core, trusted_clock=lambda: payload["auth_context"]["trusted_host_now"], diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 9d0c927..9926773 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -1118,6 +1118,8 @@ def call(document: dict[str, Any]) -> Mapping[str, Any]: "operation_token": record["operation_token"], "destination_binding_digest": record["destination_binding_digest"], "credential": credential, + "route_configuration_generation": record["route_configuration_generation"], + "handler_reference": copy.deepcopy(record["handler_reference"]), } handler = self._verified_handler( record["handler_reference"], record["destination_binding_digest"] diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index c76cea8..1fa500f 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -660,3 +660,16 @@ def other(*args): host.dispatch(root, request["effect_id"], credential="test-credential", **context) assert calls == [] assert host.snapshot(root) == before + + +@pytest.mark.parametrize("member", ["__init__", "__repr__", "__eq__", "__setattr__", "invoke"]) +def test_handler_fixture_verifier_rejects_changed_generated_or_executable_code(monkeypatch, member): + from conformance.effects_adapter import _loaded_handler, _verified_fixture_handler + + module = _loaded_handler() + assert _verified_fixture_handler(module) + if member == "invoke": + monkeypatch.setattr(module, member, lambda *args: {}) + else: + monkeypatch.setattr(module.NativeReply, member, lambda *args: None) + assert not _verified_fixture_handler(module) From 547b73a144cb096e9b44b28c5f59e3ac7da5607a Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 06:33:25 +0000 Subject: [PATCH 19/30] Prevent verified native handler method shadowing --- src/determa/state/effects.py | 24 +++++++++++++++++------- tests/test_committed_effects.py | 20 ++++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 9926773..2d95e86 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -12,6 +12,7 @@ import re import sqlite3 from collections.abc import Callable, Mapping +from dataclasses import dataclass from pathlib import Path from typing import Any, TypeVar, cast @@ -457,6 +458,7 @@ def _issue_effect_claim( return copy.deepcopy(claim) +@dataclass(frozen=True, slots=True, init=False) class VerifiedNativeHandler: """An exact native-handler instance installed through the public registry. @@ -464,14 +466,19 @@ class VerifiedNativeHandler: dependency closure. This handle never promotes provider claims into proof. """ + _registry: ExtensionRegistry + _configured: ConfiguredExtension + _reference: dict[str, Any] + _configuration: dict[str, Any] + def __init__(self, registry: ExtensionRegistry, configured: ConfiguredExtension) -> None: - self._registry = registry - self._configured = configured + object.__setattr__(self, "_registry", registry) + object.__setattr__(self, "_configured", configured) descriptor, _provider, _instance, _evaluator = registry._bound(configured) if descriptor["category"] != "native_handler": raise EffectError("host_capability_mismatch") - self._reference = copy.deepcopy(descriptor["provider_reference"]) - self._configuration = copy.deepcopy(configured._configuration) + object.__setattr__(self, "_reference", copy.deepcopy(descriptor["provider_reference"])) + object.__setattr__(self, "_configuration", copy.deepcopy(configured._configuration)) destination = self._configuration.get("destination_binding_digest") if type(destination) is not str or not re.fullmatch(r"sha256:[0-9a-f]{64}", destination): raise EffectError("host_capability_mismatch") @@ -503,7 +510,9 @@ def invoke( metadata: Mapping[str, Any], attempt: Mapping[str, Any], ) -> Mapping[str, Any]: - method, instance = self.verify(reference, metadata["destination_binding_digest"]) + method, instance = VerifiedNativeHandler.verify( + self, reference, metadata["destination_binding_digest"] + ) return cast(Mapping[str, Any], method(instance, payload, metadata, attempt)) @@ -544,7 +553,7 @@ def _verified_handler( or self.handler is not self._installed_handler ): raise EffectError("host_capability_mismatch") - self.handler.verify(reference, destination) + VerifiedNativeHandler.verify(self.handler, reference, destination) return self.handler def _connect(self) -> sqlite3.Connection: @@ -1124,7 +1133,8 @@ def call(document: dict[str, Any]) -> Mapping[str, Any]: handler = self._verified_handler( record["handler_reference"], record["destination_binding_digest"] ) - result = handler.invoke( + result = VerifiedNativeHandler.invoke( + handler, record["handler_reference"], copy.deepcopy(intent["payload"]), metadata, diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 1fa500f..c07ec00 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -673,3 +673,23 @@ def test_handler_fixture_verifier_rejects_changed_generated_or_executable_code(m else: monkeypatch.setattr(module.NativeReply, member, lambda *args: None) assert not _verified_fixture_handler(module) + + +@pytest.mark.parametrize("method", ["invoke", "verify"]) +def test_verified_native_handle_cannot_shadow_its_trusted_methods(tmp_path, method): + host, root, request, context = host_fixture(tmp_path) + record = host.snapshot(root)["journal"]["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + calls = [] + host = installed_test_handler(host, lambda *args: calls.append(args) or {"accepted": True}) + host.trusted_clock = lambda: "0" + with pytest.raises((AttributeError, TypeError)): + setattr(host.handler, method, lambda *args: {"unverified": True}) + with pytest.raises(AttributeError): + object.__setattr__(host.handler, method, lambda *args: {"unverified": True}) + assert host.dispatch(root, request["effect_id"], credential="test-credential", **context) == { + "accepted": True + } + assert len(calls) == 1 From fa1193e5dfc92a16be174bf915d9db180a72744f Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 10:06:33 +0000 Subject: [PATCH 20/30] Recheck trusted worker clock through native result commit --- src/determa/state/effects.py | 24 +++++++++++++ tests/test_committed_effects.py | 62 ++++++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 2d95e86..460be38 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -769,6 +769,17 @@ def _transact( document: dict[str, Any] = json.loads(row[0]) validate_journal(document["checkpoint"], document["journal"]) restore_execution_checkpoint_v1(document["checkpoint"], self.resolver) + expiry_guard: tuple[Callable[[], str], int] | None = None + if worker_guard is not None: + effect_id, _principal, _fence, _trusted_now = worker_guard + record = _record(document["journal"], effect_id) + if record["invocation_state"] in {"leased", "outcome_recorded"}: + claim = document["claims"].get(effect_id) + clock = self.trusted_clock + if clock is None or claim is None: + raise EffectError("stale_attempt_fence") + expiry_guard = (clock, _now(claim["expires_at"])) + self._check_live_clock(*expiry_guard) ledger = None if self.authority_scope is not None: authority = connection.execute( @@ -849,9 +860,22 @@ def _transact( "UPDATE determa_committed_effects SET document = ? WHERE root_instance_id = ?", (canonical_bytes(document), root), ) + if expiry_guard is not None: + # Keep the original deadline even when this transaction closes + # or revokes the claim. Worker rights must still hold at commit. + self._check_live_clock(*expiry_guard) connection.commit() return value + @staticmethod + def _check_live_clock(clock: Callable[[], str], deadline: int) -> None: + try: + now = _now(clock()) + except Exception as exc: + raise EffectError("stale_attempt_fence") from exc + if now >= deadline: + raise EffectError("stale_attempt_fence") + def produce( self, root: str, diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index c07ec00..f5c2fad 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -93,7 +93,12 @@ def host_fixture(tmp_path, observer=None): bundle = load_bundle((CASE / "machine.yaml").read_text()) resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) host = SQLiteCommittedEffectHost( - tmp_path / "effects.sqlite", resolver, {}, lambda *_: {}, core_observer=observer + tmp_path / "effects.sqlite", + resolver, + {}, + lambda *_: {}, + core_observer=observer, + trusted_clock=lambda: "0", ) host.setup_schema() checkpoint = read("pending-checkpoint.json") @@ -168,7 +173,9 @@ def test_invalid_result_payload_is_rejected_before_journal_or_core_mutation(tmp_ def test_payload_pointer_inserts_pinned_token_before_declared_input_admission(tmp_path): bundle = load_bundle((CASE / "machine.yaml").read_text()) resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) - host = SQLiteCommittedEffectHost(tmp_path / "pointer.sqlite", resolver, {}, lambda *_: {}) + host = SQLiteCommittedEffectHost( + tmp_path / "pointer.sqlite", resolver, {}, lambda *_: {}, trusted_clock=lambda: "0" + ) host.setup_schema() checkpoint = read("pending-checkpoint.json") journal = read("data/leased-journal.json") @@ -208,6 +215,45 @@ def test_expired_worker_cannot_admit_recorded_outcome_but_host_recovery_can(tmp_ ) +def test_worker_result_crossing_expiry_rolls_back_outcome_before_commit(tmp_path, monkeypatch): + host, root, request, context = host_fixture(tmp_path) + current_time = ["0"] + host.trusted_clock = lambda: current_time[0] + before = host.snapshot(root) + submit = host._submit + observed = [] + + def expires_during_submission(*args): + response = submit(*args) + observed.append(args[0]["journal"]["effect_records"][0]["invocation_state"]) + current_time[0] = read("data/active-claim.json")["expires_at"] + return response + + monkeypatch.setattr(host, "_submit", expires_during_submission) + assert host.submit_result(root, request, **context)["error_code"] == "stale_attempt_fence" + assert observed == ["outcome_recorded"] + assert host.snapshot(root) == before + + +@pytest.mark.parametrize("clock_state", ["missing", "unavailable", "invalid", "expired"]) +def test_worker_result_requires_current_host_clock_not_request_time(tmp_path, clock_state): + host, root, request, context = host_fixture(tmp_path) + before = host.snapshot(root) + + def unavailable(): + raise RuntimeError("clock unavailable") + + host.trusted_clock = { + "missing": None, + "unavailable": unavailable, + "invalid": lambda: "-0", + "expired": lambda: read("data/active-claim.json")["expires_at"], + }[clock_state] + assert context["trusted_now"] == "0" + assert host.submit_result(root, request, **context)["error_code"] == "stale_attempt_fence" + assert host.snapshot(root) == before + + @pytest.mark.parametrize("boundary", ["frozen", "wrong_epoch", "wrong_scope", "unissued_claim"]) def test_authority_seed_cannot_bypass_scope_or_claim_guard(tmp_path, boundary): from determa.state.authority import SQLiteLocalAuthority @@ -611,10 +657,16 @@ def test_dispatch_crossing_lease_expiry_preserves_unresolved_work_for_recovery(t {key: record[key] for key in ("handler_reference", "destination_binding_digest")} ) expiry = read("data/active-claim.json")["expires_at"] - readings = iter(["0", expiry]) - host.trusted_clock = lambda: next(readings) + current_time = ["0"] + host.trusted_clock = lambda: current_time[0] calls = [] - host = installed_test_handler(host, lambda *args: calls.append(args) or {"accepted": True}) + + def external_acceptance(*args): + calls.append(args) + current_time[0] = expiry + return {"accepted": True} + + host = installed_test_handler(host, external_acceptance) before = host.snapshot(root) with pytest.raises(EffectError, match="stale_attempt_fence"): host.dispatch(root, request["effect_id"], credential="test-credential", **context) From 46cab48df07b5055043979b364808af34e48c121 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 10:09:43 +0000 Subject: [PATCH 21/30] Bind generic worker claims to an installed colocated journal Unfinished integration checkpoint: native authority/effects regressions pass; full composed conformance and independent review remain pending. --- conformance/authority_adapter.py | 30 ++++++ src/determa/state/authority.py | 155 ++++++++++++++++++++++++++++++- tests/test_committed_effects.py | 74 +++++++++++++++ 3 files changed, 258 insertions(+), 1 deletion(-) diff --git a/conformance/authority_adapter.py b/conformance/authority_adapter.py index 4886cc3..eea51ad 100644 --- a/conformance/authority_adapter.py +++ b/conformance/authority_adapter.py @@ -139,6 +139,34 @@ def _seed(ledger: dict[str, Any]) -> SQLiteLocalAuthority: global _INITIAL_BINDING # These trusted protocol fixtures contain opaque mutations, not native rows. ledger = {**ledger, "native_checkpoint_bytes": []} + if _WORKER_MODE: + # Explicit trusted fixture installation, independent of the incoming + # request and expected result. These §18 vectors use this fixed work + # identity rather than a §19 native checkpoint/effect journal. + ledger["authority_effect_bindings"] = [ + { + "root_instance_id": "root-7", + "work_kind": "effect", + "work_identity": "effect-17", + "operation_token": "token-17", + "participant": "authority_journal", + } + for entry in ledger["journal_entries"] + if entry["work_identity"] == "effect-17" + ] + ledger["authority_effect_records"] = [ + { + **entry, + "state": "leased" + if any( + claim["work_identity"] == entry["work_identity"] and claim["state"] == "active" + for claim in ledger["active_claims"] + ) + else "unclaimed", + } + for entry in ledger["journal_entries"] + if entry["work_identity"] == "effect-17" + ] # A fresh test database for each independent vector. Production never # removes an allocated scope marker from a live authority domain. for suffix in ("", "-wal", "-shm"): @@ -176,6 +204,8 @@ def _observed_ledger(authority: SQLiteLocalAuthority) -> dict[str, Any] | None: "effect_claim_history", "native_effect_roots", "native_effect_work", + "authority_effect_bindings", + "authority_effect_records", } } diff --git a/src/determa/state/authority.py b/src/determa/state/authority.py index 0147357..616c9b2 100644 --- a/src/determa/state/authority.py +++ b/src/determa/state/authority.py @@ -545,7 +545,7 @@ def profile_descriptor( ("journal", "journal-1", Path(__file__).with_name("effects.py")), ("worker", "worker-1", Path(__file__)), ): - closure = source.read_bytes() + closure = _authority_closure() if role == "journal" else source.read_bytes() participant = { "role": role, "provider_reference": { @@ -699,6 +699,83 @@ def allocate( } return self._insert_ledger(ledger) + def register_effect_work( + self, + scope_identity: str, + owner_principal: str, + authority_epoch: str, + root_instance_id: str, + work_identity: str, + operation_token: str, + ) -> None: + """Install new host-owned effect work in the colocated authority journal. + + This trusted host setup binds identity before any worker request. It + cannot adopt an existing bare journal entry or replace a native effect. + """ + if not self.worker_fencing or any( + type(value) is not str or not value + for value in ( + scope_identity, + owner_principal, + authority_epoch, + root_instance_id, + work_identity, + operation_token, + ) + ): + raise ValueError("host_capability_mismatch") + with self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + "SELECT ledger FROM determa_scope_authority WHERE scope_identity = ?", + (scope_identity,), + ).fetchone() + if row is None: + raise ValueError("unauthorized_scope") + ledger = _parse(row[0]) + if ( + ledger["owner_principal"] != owner_principal + or ledger["authority_epoch"] != authority_epoch + or ledger["state"] != "active" + or root_instance_id not in ledger["roots"] + ): + raise ValueError("stale_scope_authority") + binding = { + "root_instance_id": root_instance_id, + "work_kind": "effect", + "work_identity": work_identity, + "operation_token": operation_token, + "participant": "authority_journal", + } + bindings = ledger.setdefault("authority_effect_bindings", []) + existing = next( + (item for item in bindings if item["work_identity"] == work_identity), None + ) + if existing is not None: + if existing != binding: + raise ValueError("invalid_host_request") + return + if root_instance_id in ledger.get("native_effect_roots", []) or any( + item["work_identity"] == work_identity + for item in ledger["journal_entries"] + + ledger.get("native_effect_work", []) + + ledger["active_claims"] + + ledger.get("effect_claim_history", []) + ): + raise ValueError("host_capability_mismatch") + bindings.append(binding) + ledger.setdefault("authority_effect_records", []).append( + {"work_identity": work_identity, "attempt_fence": "0", "state": "unclaimed"} + ) + ledger["journal_entries"].append({"work_identity": work_identity, "attempt_fence": "0"}) + ledger["scope_generation"] = str(int(ledger["scope_generation"]) + 1) + connection.execute( + "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", + (_compact(ledger), scope_identity), + ) + connection.commit() + def _insert_ledger(self, ledger: Mapping[str, Any]) -> bool: """Seed trusted ledger storage; profile fixtures use this private seam.""" scope = ledger["scope_identity"] @@ -786,6 +863,46 @@ def check_worker_claim( or now >= int(stored["expires_at"]) ): return False + binding = next( + ( + item + for item in ledger.get("authority_effect_bindings", []) + if item["work_identity"] == stored["work_identity"] + ), + None, + ) + if binding is not None: + record = next( + ( + item + for item in ledger.get("authority_effect_records", []) + if item["work_identity"] == stored["work_identity"] + ), + None, + ) + if ( + record is None + or record["state"] != "leased" + or record["attempt_fence"] != stored["attempt_fence"] + or any( + binding[key] != stored[key] + for key in ("root_instance_id", "work_kind", "operation_token") + ) + ): + return False + else: + from .effects import EffectError, _validate_authority_pair + + try: + native = connection.execute( + "SELECT document FROM determa_committed_effects WHERE root_instance_id = ?", + (stored["root_instance_id"],), + ).fetchone() + if native is None: + return False + _validate_authority_pair(ledger, _parse(bytes(native[0]).decode())) + except (EffectError, sqlite3.Error, ValueError): + return False if phase == "dispatch" and on_dispatch is not None: on_dispatch() return True @@ -1045,6 +1162,42 @@ def _perform( native_update = (canonical_bytes(native), native_root) if native_owned and native_update is None: return _compact(_result(request, ledger, "host_capability_mismatch")) + if native_update is None: + bindings = [ + item + for item in ledger.get("authority_effect_bindings", []) + if item["work_identity"] == arguments["work_identity"] + ] + records = [ + item + for item in ledger.get("authority_effect_records", []) + if item["work_identity"] == arguments["work_identity"] + ] + if len(bindings) != 1 or len(records) != 1: + return _compact(_result(request, ledger, "host_capability_mismatch")) + binding, record = bindings[0], records[0] + if binding.get("participant") != "authority_journal": + return _compact(_result(request, ledger, "host_capability_mismatch")) + if any( + binding[key] != arguments[key] + for key in ("root_instance_id", "work_kind", "operation_token") + ): + return _compact(_result(request, ledger, "invalid_host_request")) + history = [ + claim + for claim in ledger["active_claims"] + + ledger.get("effect_claim_history", []) + if claim["work_identity"] == arguments["work_identity"] + ] + if ( + record["attempt_fence"] != entry["attempt_fence"] + or record["state"] != "unclaimed" + or any(int(claim["attempt_fence"]) >= int(fence) for claim in history) + ): + return _compact(_result(request, ledger, "stale_attempt_fence")) + record["attempt_fence"] = fence + record["state"] = "leased" + ledger.setdefault("effect_claim_history", []).append(copy.deepcopy(new_claim)) if native_update is not None: connection.execute( "UPDATE determa_committed_effects SET document=? WHERE root_instance_id=?", diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index f5c2fad..dac00c7 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -407,6 +407,80 @@ def authority_claim_request( return json.dumps(request), invocation +def generic_authority_fixture(tmp_path): + from determa.state.authority import SQLiteLocalAuthority + + authority = SQLiteLocalAuthority( + tmp_path / "generic.sqlite", worker_fencing=True, worker_lease_nanoseconds=10 + ) + authority.setup_schema() + assert authority.allocate("scope", "owner", roots=("root", "other-root")) + authority.register_effect_work("scope", "owner", "0", "root", "effect", "token") + return authority, {"effect_id": "effect", "operation_token": "token"} + + +def test_generic_claim_commits_bound_journal_and_receipt_atomically(tmp_path): + authority, record = generic_authority_fixture(tmp_path) + request, invocation = authority_claim_request(authority, "scope", "root", record) + before = authority.inspect("scope") + assert authority._perform(request, invocation, fault="precommit_abort") is None + assert authority.inspect("scope") == before + assert authority._perform(request, invocation, fault="drop_response_after_commit") is None + committed = authority.inspect("scope") + claim = committed["active_claims"][0] + assert committed["authority_effect_records"] == [ + {"work_identity": "effect", "attempt_fence": "1", "state": "leased"} + ] + assert committed["effect_claim_history"] == [claim] + assert json.loads(authority.perform(request, invocation))["claim"] == claim + assert authority.inspect("scope") == committed + assert authority.check_worker_claim(claim, "worker-a", "0", phase="dispatch") + assert not authority.check_worker_claim(claim, "worker-a", "10", phase="dispatch") + + +@pytest.mark.parametrize("change", ["root", "token", "binding", "record", "fence"]) +def test_generic_claim_cannot_rebind_work_or_fall_back_from_missing_participant(tmp_path, change): + authority, record = generic_authority_fixture(tmp_path) + overrides = {} + if change == "root": + overrides["root_instance_id"] = "other-root" + elif change == "token": + overrides["operation_token"] = "other-token" + else: + with authority._connect() as connection: + ledger = authority.inspect("scope") + if change == "binding": + ledger["authority_effect_bindings"] = [] + elif change == "record": + ledger["authority_effect_records"] = [] + else: + ledger["authority_effect_records"][0]["attempt_fence"] = "2" + connection.execute( + "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", + (json.dumps(ledger), "scope"), + ) + request, invocation = authority_claim_request(authority, "scope", "root", record, **overrides) + before = authority.inspect("scope") + assert json.loads(authority.perform(request, invocation))["status"] == "rejected" + assert authority.inspect("scope") == before + + +def test_generic_registration_preserves_binding_and_expiry_does_not_prove_retry_safe(tmp_path): + authority, record = generic_authority_fixture(tmp_path) + before = authority.inspect("scope") + authority.register_effect_work("scope", "owner", "0", "root", "effect", "token") + assert authority.inspect("scope") == before + with pytest.raises(ValueError, match="invalid_host_request"): + authority.register_effect_work("scope", "owner", "0", "other-root", "effect", "token") + request, invocation = authority_claim_request(authority, "scope", "root", record) + assert json.loads(authority.perform(request, invocation))["status"] == "accepted" + request, invocation = authority_claim_request(authority, "scope", "root", record, "retry", "1") + invocation["trusted_clock_now"] = "10" + before = authority.inspect("scope") + assert json.loads(authority.perform(request, invocation))["error_code"] == "stale_attempt_fence" + assert authority.inspect("scope") == before + + def test_authority_claim_atomically_updates_native_journal_and_replays_once(tmp_path): authority, host, scope, root, record = authority_effect_fixture(tmp_path) request, invocation = authority_claim_request(authority, scope, root, record) From 3b5d1c9a98e8691acd754357d1f3645f387f3ad8 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 10:11:37 +0000 Subject: [PATCH 22/30] Require exact native work binding before worker dispatch guard --- src/determa/state/authority.py | 26 +++++++++++++++++++++++++- tests/test_committed_effects.py | 25 +++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/determa/state/authority.py b/src/determa/state/authority.py index 616c9b2..850daf7 100644 --- a/src/determa/state/authority.py +++ b/src/determa/state/authority.py @@ -894,13 +894,37 @@ def check_worker_claim( from .effects import EffectError, _validate_authority_pair try: + bindings = [ + item + for item in ledger.get("native_effect_work", []) + if item["work_identity"] == stored["work_identity"] + ] + if len(bindings) != 1 or any( + bindings[0][key] != stored[key] + for key in ("root_instance_id", "work_kind", "operation_token") + ): + return False native = connection.execute( "SELECT document FROM determa_committed_effects WHERE root_instance_id = ?", (stored["root_instance_id"],), ).fetchone() if native is None: return False - _validate_authority_pair(ledger, _parse(bytes(native[0]).decode())) + document = _parse(bytes(native[0]).decode()) + records = [ + item + for item in document["journal"]["effect_records"] + if item["effect_id"] == stored["work_identity"] + ] + if ( + len(records) != 1 + or records[0]["operation_token"] != stored["operation_token"] + or records[0]["attempt_fence"] != stored["attempt_fence"] + or records[0]["invocation_state"] != "leased" + or document["claims"].get(stored["work_identity"]) != stored + ): + return False + _validate_authority_pair(ledger, document) except (EffectError, sqlite3.Error, ValueError): return False if phase == "dispatch" and on_dispatch is not None: diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index dac00c7..b36cf25 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -481,6 +481,31 @@ def test_generic_registration_preserves_binding_and_expiry_does_not_prove_retry_ assert authority.inspect("scope") == before +def test_native_worker_guard_refuses_unbound_claim_even_with_valid_native_root(tmp_path): + authority, host, scope, root, record = authority_effect_fixture(tmp_path) + request, invocation = authority_claim_request(authority, scope, root, record) + claim = json.loads(authority.perform(request, invocation))["claim"] + assert authority.check_worker_claim(claim, "worker-a", "0", phase="dispatch") + unbound = {**claim, "work_identity": "unbound-work", "operation_token": "unbound-token"} + with authority._connect() as connection: + ledger = authority.inspect(scope) + ledger["journal_entries"].append( + {"work_identity": unbound["work_identity"], "attempt_fence": unbound["attempt_fence"]} + ) + ledger["active_claims"].append(unbound) + connection.execute( + "UPDATE determa_scope_authority SET ledger = ? WHERE scope_identity = ?", + (json.dumps(ledger), scope), + ) + before = host.snapshot(root) + calls = [] + assert not authority.check_worker_claim( + unbound, "worker-a", "0", phase="dispatch", on_dispatch=lambda: calls.append("called") + ) + assert calls == [] + assert host.snapshot(root) == before + + def test_authority_claim_atomically_updates_native_journal_and_replays_once(tmp_path): authority, host, scope, root, record = authority_effect_fixture(tmp_path) request, invocation = authority_claim_request(authority, scope, root, record) From bc8de88f3cb0406ab57ba6d89ecdd0d53171c960 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 10:22:10 +0000 Subject: [PATCH 23/30] Verify exact Python 3.13 generated class metadata --- conformance/effects_adapter.py | 6 ++++++ tests/test_committed_effects.py | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/conformance/effects_adapter.py b/conformance/effects_adapter.py index 5397cd8..0b757f5 100644 --- a/conformance/effects_adapter.py +++ b/conformance/effects_adapter.py @@ -311,6 +311,12 @@ def _verified_fixture_handler(module: Any) -> bool: frozen=True, ) expected.__module__ = module.__name__ + if sys.version_info >= (3, 13): + # CPython adds these non-executable class-body metadata fields. A class + # built by make_dataclass has no compiled body; reproduce only the exact + # values proved by this allowlisted declaration, not arbitrary metadata. + expected.__firstlineno__ = declaration.decorator_list[0].lineno + expected.__static_attributes__ = () actual = module.NativeReply if ( type(actual) is not type diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index b36cf25..830a899 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -826,6 +826,18 @@ def test_handler_fixture_verifier_rejects_changed_generated_or_executable_code(m assert not _verified_fixture_handler(module) +@pytest.mark.parametrize( + ("member", "value"), [("__firstlineno__", 999), ("__static_attributes__", ("unexpected",))] +) +def test_handler_fixture_verifier_checks_compiler_metadata_exactly(monkeypatch, member, value): + from conformance.effects_adapter import _loaded_handler, _verified_fixture_handler + + module = _loaded_handler() + assert _verified_fixture_handler(module) + monkeypatch.setattr(module.NativeReply, member, value, raising=False) + assert not _verified_fixture_handler(module) + + @pytest.mark.parametrize("method", ["invoke", "verify"]) def test_verified_native_handle_cannot_shadow_its_trusted_methods(tmp_path, method): host, root, request, context = host_fixture(tmp_path) From 77925d2c53136302740a815290a4556c4ae572ee Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 10:26:01 +0000 Subject: [PATCH 24/30] Require registry-installed handler handles and document effect guarantees --- README.md | 37 ++++++++++++++++++++++++++++---- conformance/authority_adapter.py | 2 +- conformance/effects_adapter.py | 2 +- src/determa/state/effects.py | 4 +++- tests/test_committed_effects.py | 23 ++++++++++++++------ 5 files changed, 55 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 65e330c..b1fb4e7 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ This implementation supports Determa State `format: 1` at specification commit `86bb88dd21cb1f799eefe5020b6e49dabf6e7225`. Correctness is determined by 99 format-1 core cases, 162 version-1 vectors, 142 durable-host vectors, 49 core inspection vectors, 47 extension negotiation vectors, 7 native guard-provider -inspection vectors, 30 runtime-provider vectors, and 476 manifest artifacts at -conformance commit `ce6c94ee1929597689403080d1790e2f48f8a5fb`. The native +inspection vectors, 30 runtime-provider vectors, and 484 manifest artifacts at +conformance commit `7f09321fb483a22eb677a4342f8d9537a7a18e82`. The native runtime-provider profile is optional and is exercised through its production adapter. The package metadata is `0.3.0`. Artifact and checkpoint schema version 1 is the @@ -391,11 +391,40 @@ health and the same store instance used by the host. Recheck it before relying o the guarded profile. A frozen scope retains fencing and inventory claims while checkpoint writes remain refused. -The reference boundary does not issue worker claims or retirement grants and -reports `worker_fencing: false` and `safe_relocation: false`. A copied database, +The default checkpoint-only configuration reports `worker_fencing: false`. +An explicitly configured colocated worker journal can issue local claims with +`worker_fencing=True`; it binds the root, effect identity and operation token +before issuance, and commits journal state, claim, generation and receipt together. +The reference topology does not issue retirement grants and reports +`safe_relocation: false`. A copied database, new endpoint, or timeout cannot activate another owner. The pure in-memory API needs no scope authority. +## Optional committed native effects + +`determa.state.effects.SQLiteCommittedEffectHost` implements the optional local +SQLite effect journal. Install a `VerifiedNativeHandler` through the ordinary +`ExtensionRegistry` with a trusted source verifier, exact provider reference, +configured instance and destination binding. Plain callbacks cannot be installed +as handlers. A host without a handler may restore or reconcile committed data; +producing or dispatching external work requires the verified installed handle. +SDK objects can remain private inside the handler; boundary payloads use declared +portable typed values. + +The host commits a pinned effect intent before dispatch. Worker calls use an +independently authenticated principal, current scope/epoch/attempt claim and an +explicit `trusted_clock` callback. Clock failures or expiry fail closed, including +expiry during outcome recording before its native commit. An expired external +attempt remains unresolved and requires destination idempotency or reconciliation; +lease expiry does not prove the call was undone. + +Authenticated outcome recording commits before result-event admission. A crash +between those transactions retains the outcome, and `recover` admits it without +calling the provider again. Each recoverable effect commits separately. Duplicate +results retain exact first responses; conflicting or stale work is refused. +Delivery acknowledgement and business outcome remain separate. The optional host +does not create a background worker or claim arbitrary external exactly-once work. + ## Develop ```sh diff --git a/conformance/authority_adapter.py b/conformance/authority_adapter.py index eea51ad..d5c832f 100644 --- a/conformance/authority_adapter.py +++ b/conformance/authority_adapter.py @@ -175,7 +175,7 @@ def _seed(ledger: dict[str, Any]) -> SQLiteLocalAuthority: authority.setup_schema() AuthoritySQLiteExecutionStore(authority, "scope-42", "owner-1", "2").setup_schema() if _WORKER_MODE: - SQLiteCommittedEffectHost(_PATH, MemoryArtifactResolver(), {}, lambda *_: {}).setup_schema() + SQLiteCommittedEffectHost(_PATH, MemoryArtifactResolver(), {}, None).setup_schema() if not authority._insert_ledger(_baseline_ledger()): raise RuntimeError("fixture scope already allocated") _INITIAL_BINDING, authority = _binding_live() diff --git a/conformance/effects_adapter.py b/conformance/effects_adapter.py index 0b757f5..4d933b0 100644 --- a/conformance/effects_adapter.py +++ b/conformance/effects_adapter.py @@ -150,7 +150,7 @@ def _reset_authority() -> tuple[SQLiteLocalAuthority, dict[str, Any], dict[str, authority = SQLiteLocalAuthority(_PATH, worker_fencing=True, worker_lease_nanoseconds=1) authority.setup_schema() AuthoritySQLiteExecutionStore(authority, _AUTHORITY_SCOPE, "owner-1", "3").setup_schema() - SQLiteCommittedEffectHost(_PATH, MemoryArtifactResolver(), {}, lambda *_: {}).setup_schema() + SQLiteCommittedEffectHost(_PATH, MemoryArtifactResolver(), {}, None).setup_schema() if not authority._insert_ledger(_authority_ledger()): raise RuntimeError("effect scope allocation failed") registry = bundled_extension_registry(include_postgresql=False) diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 460be38..088c98a 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -530,12 +530,14 @@ def __init__( path: str | Path, resolver: ArtifactResolver, route: Mapping[str, Any], - handler: VerifiedNativeHandler | Callable[..., Mapping[str, Any]], + handler: VerifiedNativeHandler | None, *, authority_scope: str | None = None, core_observer: Callable[[str, str, Mapping[str, Any]], None] | None = None, trusted_clock: Callable[[], str] | None = None, ) -> None: + if handler is not None and type(handler) is not VerifiedNativeHandler: + raise EffectError("host_capability_mismatch") self.path = str(Path(path).resolve()) self.resolver = resolver self.route = copy.deepcopy(dict(route)) diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 830a899..9724be1 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -96,7 +96,7 @@ def host_fixture(tmp_path, observer=None): tmp_path / "effects.sqlite", resolver, {}, - lambda *_: {}, + None, core_observer=observer, trusted_clock=lambda: "0", ) @@ -174,7 +174,7 @@ def test_payload_pointer_inserts_pinned_token_before_declared_input_admission(tm bundle = load_bundle((CASE / "machine.yaml").read_text()) resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) host = SQLiteCommittedEffectHost( - tmp_path / "pointer.sqlite", resolver, {}, lambda *_: {}, trusted_clock=lambda: "0" + tmp_path / "pointer.sqlite", resolver, {}, None, trusted_clock=lambda: "0" ) host.setup_schema() checkpoint = read("pending-checkpoint.json") @@ -271,7 +271,7 @@ def test_authority_seed_cannot_bypass_scope_or_claim_guard(tmp_path, boundary): claim["scope_authority_epoch"] = "0" epoch = "9" if boundary == "wrong_epoch" else "0" host = SQLiteCommittedEffectHost( - authority.path, resolver, {"authority_epoch": epoch}, lambda *_: {}, authority_scope=scope + authority.path, resolver, {"authority_epoch": epoch}, None, authority_scope=scope ) host.setup_schema() if boundary == "frozen": @@ -339,7 +339,7 @@ def test_seed_cannot_rewind_an_issued_fence_or_remove_its_claim(tmp_path, journa ) connection.commit() host = SQLiteCommittedEffectHost( - authority.path, resolver, {"authority_epoch": "0"}, lambda *_: {}, authority_scope=scope + authority.path, resolver, {"authority_epoch": "0"}, None, authority_scope=scope ) host.setup_schema() with pytest.raises(EffectError, match="stale_attempt_fence"): @@ -364,7 +364,7 @@ def authority_effect_fixture(tmp_path): authority.setup_schema() assert authority.allocate(scope, "owner", roots=(root,)) host = SQLiteCommittedEffectHost( - authority.path, resolver, {"authority_epoch": "0"}, lambda *_: {}, authority_scope=scope + authority.path, resolver, {"authority_epoch": "0"}, None, authority_scope=scope ) host.setup_schema() host.seed(checkpoint, journal) @@ -726,7 +726,7 @@ def test_multi_effect_recovery_commits_one_journal_revision_per_effect(tmp_path) journal["checkpoint_revision"] = checkpoint["revision"] journal["checkpoint_digest"] = checkpoint["execution_checkpoint_digest"] journal = seal_journal(journal) - host = SQLiteCommittedEffectHost(tmp_path / "two.sqlite", original.resolver, {}, lambda *_: {}) + host = SQLiteCommittedEffectHost(tmp_path / "two.sqlite", original.resolver, {}, None) host.setup_schema() host.seed(checkpoint, journal) for record in journal["effect_records"]: @@ -856,3 +856,14 @@ def test_verified_native_handle_cannot_shadow_its_trusted_methods(tmp_path, meth "accepted": True } assert len(calls) == 1 + + +def test_plain_callback_handler_is_refused_before_opening_storage(tmp_path): + path = tmp_path / "unverified.sqlite" + calls = [] + with pytest.raises(EffectError, match="host_capability_mismatch"): + SQLiteCommittedEffectHost( + path, MemoryArtifactResolver(), {}, lambda *args: calls.append(args) or {} + ) + assert calls == [] + assert not path.exists() From 0dad76bb2da7d1b5b0265e52860bea1c6ecd104b Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 10:38:32 +0000 Subject: [PATCH 25/30] fix: authenticate committed effect recovery and closed requests --- src/determa/state/effects.py | 278 ++++++++++++++++++++++++-------- tests/test_committed_effects.py | 197 +++++++++++++++++++++- 2 files changed, 407 insertions(+), 68 deletions(-) diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 088c98a..c3bbae9 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -79,6 +79,18 @@ def _journal_validator() -> Any: return Draft202012Validator(source, registry=registry) +def _validate_request(request: Mapping[str, Any], name: str) -> dict[str, Any]: + from jsonschema import Draft202012Validator + + if not isinstance(request, Mapping): + raise EffectError("invalid_host_request") + normalized = copy.deepcopy(dict(request)) + source = json.loads((Path(__file__).parent / "data" / name).read_text()) + if not Draft202012Validator(source, registry=_schema_registry()).is_valid(normalized): + raise EffectError("invalid_host_request") + return normalized + + def _now(value: str) -> int: if type(value) is not str or not _NANOSECONDS.fullmatch(value): raise EffectError("stale_attempt_fence") @@ -110,7 +122,106 @@ def _bump(journal: dict[str, Any], checkpoint: Mapping[str, Any] | None = None) journal["host_effect_journal_digest"] = journal_digest(journal) -def validate_journal(checkpoint: Mapping[str, Any], journal: Mapping[str, Any]) -> None: +def _result_delivery_evidence( + checkpoint: Mapping[str, Any], + record: Mapping[str, Any], + kind: str, + payload: Any, + target: Mapping[str, Any], +) -> dict[str, Any]: + """Build pinned admission bytes without consulting current runtime liveness.""" + mapping = _mapping(record, kind) + event_id = hash_value( + ["determa-effect-result-event-1", record["effect_id"], mapping["result_slot"]] + ) + envelope = { + "event": mapping["event"], + "event_id": event_id, + "cause_id": event_id, + "source": {"host": True}, + "target": copy.deepcopy(dict(target)), + "payload": copy.deepcopy(payload), + } + location = mapping["operation_token_location"] + if location is not None: + if location["kind"] == "correlation_id": + envelope["correlation_id"] = record["operation_token"] + else: + try: + logical = decoded_typed_value(payload) + pointer = location["pointer"] + if not pointer.startswith("/") or re.search(r"~(?![01])", pointer): + raise ValueError("invalid token pointer") + parts = [ + part.replace("~1", "/").replace("~0", "~") for part in pointer[1:].split("/") + ] + parent = logical + for part in parts[:-1]: + parent = parent[part] + if not isinstance(parent, dict) or ( + parts[-1] in parent and not isinstance(parent[parts[-1]], str) + ): + raise ValueError("token field must be a string") + parent[parts[-1]] = record["operation_token"] + envelope["payload"] = typed_value(logical) + except (KeyError, TypeError, ValueError) as exc: + raise EffectError("invalid_host_request") from exc + from .queueing import _entry_digest + + return { + "delivery_mode": "input", + "envelope": envelope, + "envelope_digest": _entry_digest(checkpoint["root_instance_id"], "input", envelope), + } + + +def _pinned_result_target( + checkpoint: Mapping[str, Any], + record: Mapping[str, Any], + resolver: ArtifactResolver | None, +) -> dict[str, Any]: + target = record["target"] + origin = target["runtime_incarnation"] + aggregate = checkpoint["root_record"].get("aggregate_state") + for runtime in aggregate["runtimes"] if aggregate else []: + if runtime["runtime_id"] == target["runtime_id"] and runtime["identity_origin"] == origin: + return copy.deepcopy(runtime["target_identity"]) + root, runtime_id = checkpoint["root_instance_id"], target["runtime_id"] + if origin["kind"] == "root" and origin["root_instance_id"] == root: + return {"root": {"root_instance_id": root, "root_runtime_id": runtime_id}} + if origin["kind"] == "owned_spawned_instance": + machine = origin["definition"]["machine"] + return { + "spawned_instance": { + "root_instance_id": root, + "instance_id": runtime_id, + "machine_id": machine["machine_id"], + "machine_version": machine["machine_version"], + } + } + if origin["kind"] == "component" and resolver is not None: + from .engine import _pointer_get + from .wire import _origin_machine + + bundle, _ = _origin_machine(resolver, origin) + placement = _pointer_get(bundle.raw, origin["component_definition_pointer"]) + return { + "component": { + "root_instance_id": root, + "owner_runtime_id": origin["owner_runtime_id"], + "component_id": placement["component_id"], + "component_runtime_id": runtime_id, + "activation_sequence": origin["activation_sequence"], + } + } + raise EffectError("invalid_effect_journal") + + +def validate_journal( + checkpoint: Mapping[str, Any], + journal: Mapping[str, Any], + resolver: ArtifactResolver | None = None, +) -> None: """Reject torn or internally inconsistent journal/checkpoint pairs.""" if not _journal_validator().is_valid(journal): raise EffectError("invalid_effect_journal") @@ -128,6 +239,11 @@ def validate_journal(checkpoint: Mapping[str, Any], journal: Mapping[str, Any]) {r["effect_id"] for r in records} ): raise EffectError("invalid_effect_journal") + references = journal["operation_response_references"] + if [item["operation_id"] for item in references] != sorted( + {item["operation_id"] for item in references} + ): + raise EffectError("invalid_effect_journal") intents = { item["intent"]["effect_id"]: item["intent"] for item in checkpoint["pending_outbox_intents"] } @@ -137,10 +253,18 @@ def validate_journal(checkpoint: Mapping[str, Any], journal: Mapping[str, Any]) for item in checkpoint["terminal_outbox_records"] } ) + compact = {item["effect_id"]: item for item in checkpoint["outbox_effect_tombstones"]} + location_ids = [ + item["intent"]["effect_id"] + for item in checkpoint["pending_outbox_intents"] + checkpoint["terminal_outbox_records"] + ] + [item["effect_id"] for item in checkpoint["outbox_effect_tombstones"]] for record in records: intent = intents.get(record["effect_id"]) - if intent is None or record["intent_digest"] != outbox_intent_digest( - checkpoint["root_instance_id"], intent + tombstone = compact.get(record["effect_id"]) + if location_ids.count(record["effect_id"]) != 1 or record["intent_digest"] != ( + outbox_intent_digest(checkpoint["root_instance_id"], intent) + if intent is not None + else cast(dict[str, Any], tombstone)["intent_digest"] ): raise EffectError("invalid_effect_journal") reports = record["attempt_records"] @@ -161,6 +285,7 @@ def validate_journal(checkpoint: Mapping[str, Any], journal: Mapping[str, Any]) if state in {"result_admitted", "closed"} and ( record["admission_receipt"] is None or record["admission_receipt"]["event_id"] != record["result_event_id"] + or record["admission_receipt"] not in checkpoint["operation_receipts"] ): raise EffectError("invalid_effect_journal") outcome = record["outcome"] @@ -192,6 +317,24 @@ def validate_journal(checkpoint: Mapping[str, Any], journal: Mapping[str, Any]) ] ): raise EffectError("invalid_effect_journal") + if state in {"result_admitted", "closed"}: + try: + delivery = _result_delivery_evidence( + checkpoint, + record, + outcome["kind"], + outcome["payload"], + _pinned_result_target(checkpoint, record, resolver), + ) + receipt = record["admission_receipt"] + if ( + receipt["operation_kind"] != "acceptance" + or receipt["delivery_mode"] != "input" + or receipt["request_digest"] != delivery["envelope_digest"] + ): + raise EffectError("invalid_effect_journal") + except (ArtifactError, KeyError, TypeError, ValueError) as exc: + raise EffectError("invalid_effect_journal") from exc preclaim_cancel = ( outcome["kind"] == "cancelled" and outcome["attempt_fence"] == "0" @@ -577,7 +720,7 @@ def seed( journal: Mapping[str, Any], claim: Mapping[str, Any] | None = None, ) -> None: - validate_journal(checkpoint, journal) + validate_journal(checkpoint, journal, self.resolver) restore_execution_checkpoint_v1(checkpoint, self.resolver) document: dict[str, Any] = { "checkpoint": checkpoint, @@ -748,7 +891,7 @@ def snapshot(self, root_instance_id: str) -> dict[str, Any]: if row is None: raise EffectError("wrong_root") document: dict[str, Any] = json.loads(row[0]) - validate_journal(document["checkpoint"], document["journal"]) + validate_journal(document["checkpoint"], document["journal"], self.resolver) restore_execution_checkpoint_v1(document["checkpoint"], self.resolver) return document @@ -769,7 +912,7 @@ def _transact( if row is None: raise EffectError("wrong_root") document: dict[str, Any] = json.loads(row[0]) - validate_journal(document["checkpoint"], document["journal"]) + validate_journal(document["checkpoint"], document["journal"], self.resolver) restore_execution_checkpoint_v1(document["checkpoint"], self.resolver) expiry_guard: tuple[Callable[[], str], int] | None = None if worker_guard is not None: @@ -842,7 +985,7 @@ def _transact( raise EffectError("stale_attempt_fence") before = canonical_bytes(document) value = change(document) - validate_journal(document["checkpoint"], document["journal"]) + validate_journal(document["checkpoint"], document["journal"], self.resolver) restore_execution_checkpoint_v1(document["checkpoint"], self.resolver) if ( ledger is not None @@ -1193,20 +1336,27 @@ def submit_result( scope: str, epoch: str, trusted_now: str, - admit: bool = True, ) -> dict[str, Any]: + if ( + not isinstance(request, Mapping) + or type(request.get("effect_id")) is not str + or re.fullmatch(r"sha256:[0-9a-f]{64}", request["effect_id"]) is None + or type(request.get("attempt_fence")) is not str + or re.fullmatch(r"0|[1-9][0-9]*", request["attempt_fence"]) is None + ): + raise EffectError("invalid_host_request") effect_id, fence = request["effect_id"], request["attempt_fence"] try: - response = self._transact( + request = _validate_request(request, "effect-result-request-v1.schema.json") + response = self._record_result( root, - lambda document: self._submit( - document, request, principal, scope, epoch, trusted_now, False - ), - expected_epoch=epoch, - authority_mutation=self._mirror_authority, - worker_guard=(effect_id, principal, fence, trusted_now), + request, + principal=principal, + scope=scope, + epoch=epoch, + trusted_now=trusted_now, ) - if admit and request["outcome_kind"] in _OUTCOMES: + if request["outcome_kind"] in _OUTCOMES: # The authenticated outcome is already durable. Admission is a # separate host-owned transaction; its failure cannot erase it. def admission(document: dict[str, Any]) -> dict[str, Any]: @@ -1229,6 +1379,25 @@ def admission(document: dict[str, Any]) -> dict[str, Any]: except EffectError as exc: return _rejected_result(effect_id, fence, exc.code) + def _record_result( + self, + root: str, + request: Mapping[str, Any], + *, + principal: str, + scope: str, + epoch: str, + trusted_now: str, + ) -> dict[str, Any]: + """Private durable-outcome stage; never a completed public result.""" + return self._transact( + root, + lambda document: self._submit(document, request, principal, scope, epoch, trusted_now), + expected_epoch=epoch, + authority_mutation=self._mirror_authority, + worker_guard=(request["effect_id"], principal, request["attempt_fence"], trusted_now), + ) + def _submit( self, document: dict[str, Any], @@ -1237,7 +1406,6 @@ def _submit( scope: str, epoch: str, trusted_now: str, - admit: bool, ) -> dict[str, Any]: journal, checkpoint = document["journal"], document["checkpoint"] effect_id, fence, kind = ( @@ -1273,14 +1441,11 @@ def _submit( return copy.deepcopy(saved) if record["invocation_state"] in {"result_admitted", "closed"}: return _result_response(record, journal, checkpoint, "committed") - if record["invocation_state"] == "outcome_recorded" and admit: - self._admit(document, record) - return _result_response(record, journal, document["checkpoint"], "committed") return _result_response( record, journal, checkpoint, - "report_recorded" if kind not in _OUTCOMES else "committed", + "report_recorded" if kind not in _OUTCOMES else "outcome_recorded", next( (item for item in record["attempt_records"] if item["attempt_fence"] == fence), None, @@ -1337,14 +1502,11 @@ def _submit( ) record["invocation_state"] = "outcome_recorded" _bump(journal) - if admit: - self._admit(document, record) - return _result_response(record, journal, document["checkpoint"], "committed") + return _result_response(record, journal, document["checkpoint"], "outcome_recorded") def _result_delivery( self, checkpoint: dict[str, Any], record: dict[str, Any], kind: str, payload: Any ) -> dict[str, Any]: - mapping = _mapping(record, kind) aggregate = checkpoint["root_record"].get("aggregate_state") runtime = next( ( @@ -1360,49 +1522,9 @@ def _result_delivery( or record["target"]["root_instance_id"] != checkpoint["root_instance_id"] ): raise EffectError("effect_not_outstanding") - event_id = hash_value( - ["determa-effect-result-event-1", record["effect_id"], mapping["result_slot"]] + delivery = _result_delivery_evidence( + checkpoint, record, kind, payload, runtime["target_identity"] ) - envelope = { - "event": mapping["event"], - "event_id": event_id, - "cause_id": event_id, - "source": {"host": True}, - "target": copy.deepcopy(runtime["target_identity"]), - "payload": copy.deepcopy(payload), - } - location = mapping["operation_token_location"] - if location is not None: - if location["kind"] == "correlation_id": - envelope["correlation_id"] = record["operation_token"] - else: - try: - logical = decoded_typed_value(payload) - pointer = location["pointer"] - if not pointer.startswith("/") or re.search(r"~(?![01])", pointer): - raise ValueError("invalid token pointer") - parts = [ - part.replace("~1", "/").replace("~0", "~") - for part in pointer[1:].split("/") - ] - parent = logical - for part in parts[:-1]: - parent = parent[part] - if not isinstance(parent, dict) or ( - parts[-1] in parent and not isinstance(parent[parts[-1]], str) - ): - raise ValueError("token field must be a string") - parent[parts[-1]] = record["operation_token"] - envelope["payload"] = typed_value(logical) - except (KeyError, TypeError, ValueError) as exc: - raise EffectError("invalid_host_request") from exc - from .queueing import _entry_digest - - delivery = { - "delivery_mode": "input", - "envelope": envelope, - "envelope_digest": _entry_digest(checkpoint["root_instance_id"], "input", envelope), - } try: _preflight_checkpoint_admission( checkpoint, @@ -1482,6 +1604,28 @@ def change(document: dict[str, Any], effect_id: str = effect_id) -> None: return self.snapshot(root) def cancel(self, root: str, request: Mapping[str, Any]) -> dict[str, Any]: + if ( + not isinstance(request, Mapping) + or type(request.get("effect_id")) is not str + or re.fullmatch(r"sha256:[0-9a-f]{64}", request["effect_id"]) is None + or type(request.get("operation_id")) is not str + or not request["operation_id"] + ): + raise EffectError("invalid_host_request") + try: + request = _validate_request(request, "effect-cancellation-request-v1.schema.json") + except EffectError: + return { + "status": "rejected", + "operation_id": request["operation_id"], + "effect_id": request["effect_id"], + "cancellation": None, + "outcome": None, + "result_event_id": None, + "journal_revision": None, + "error_code": "invalid_host_request", + } + def change(document: dict[str, Any]) -> dict[str, Any]: journal = document["journal"] record = _record(journal, request["effect_id"]) diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 9724be1..bfb2138 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -205,7 +205,7 @@ def test_payload_pointer_inserts_pinned_token_before_declared_input_admission(tm def test_expired_worker_cannot_admit_recorded_outcome_but_host_recovery_can(tmp_path): host, root, request, context = host_fixture(tmp_path) - host.submit_result(root, request, admit=False, **context) + host._record_result(root, request, **context) before = host.snapshot(root) context["trusted_now"] = read("data/active-claim.json")["expires_at"] assert host.submit_result(root, request, **context)["error_code"] == "stale_attempt_fence" @@ -867,3 +867,198 @@ def test_plain_callback_handler_is_refused_before_opening_storage(tmp_path): ) assert calls == [] assert not path.exists() + + +def test_public_result_requires_admission_before_committed_response(tmp_path): + host, root, request, context = host_fixture(tmp_path) + before = host.snapshot(root) + with pytest.raises(TypeError, match="admit"): + host.submit_result(root, request, admit=False, **context) + assert host.snapshot(root) == before + staged = host._record_result(root, request, **context) + assert staged["status"] == "outcome_recorded" + assert staged["admission_receipt"] is None + response = host.submit_result(root, request, **context) + assert response["status"] == "committed" + assert response["admission_receipt"] in host.snapshot(root)["checkpoint"]["operation_receipts"] + + +@pytest.mark.parametrize("operation", ["result", "cancel"]) +@pytest.mark.parametrize( + "damage", ["extra", "missing_payload", "native_payload", "missing_identity"] +) +def test_effect_requests_fail_closed_before_mutation_or_core_call(tmp_path, operation, damage): + calls = [] + host, root, request, context = host_fixture(tmp_path, lambda *args: calls.append(args)) + if operation == "cancel": + request = read("data/cancel-request.json") + if damage == "extra": + request["unexpected"] = True + elif damage == "missing_payload": + del request["payload"] + elif damage == "native_payload": + request["payload"] = object() + else: + del request["effect_id"] + before = host.snapshot(root) + + def invoke(): + return ( + host.submit_result(root, request, **context) + if operation == "result" + else host.cancel(root, request) + ) + + if damage == "missing_identity": + with pytest.raises(EffectError, match="invalid_host_request"): + invoke() + else: + response = invoke() + assert response["status"] == "rejected" + assert response["error_code"] == "invalid_host_request" + assert calls == [] + assert host.snapshot(root) == before + + +def test_noncanonical_result_fence_is_refused_before_storage_mutation(tmp_path): + host, root, request, context = host_fixture(tmp_path) + request["attempt_fence"] = "01" + before = host.snapshot(root) + with pytest.raises(EffectError, match="invalid_host_request"): + host.submit_result(root, request, **context) + assert host.snapshot(root) == before + + +@pytest.mark.parametrize("operation_ids", [["z", "a"], ["dup", "dup"]]) +def test_journal_response_references_require_strictly_ordered_unique_ids(operation_ids): + journal = read("data/leased-journal.json") + journal["operation_response_references"] = [ + {"operation_id": name, "response_digest": "sha256:" + str(index) * 64} + for index, name in enumerate(operation_ids) + ] + with pytest.raises(EffectError, match="invalid_effect_journal"): + validate_journal(read("pending-checkpoint.json"), seal_journal(journal)) + + +def test_restoration_rejects_admitted_journal_without_checkpoint_admission(tmp_path): + host, root, request, context = host_fixture(tmp_path) + before = host.snapshot(root)["checkpoint"] + host.submit_result(root, request, **context) + journal = host.snapshot(root)["journal"] + journal["checkpoint_revision"] = before["revision"] + journal["checkpoint_digest"] = before["execution_checkpoint_digest"] + restored = SQLiteCommittedEffectHost(tmp_path / "torn.sqlite", host.resolver, {}, None) + restored.setup_schema() + with pytest.raises(EffectError, match="invalid_effect_journal"): + restored.seed(before, seal_journal(journal)) + with restored._connect() as connection: + assert connection.execute("SELECT COUNT(*) FROM determa_committed_effects").fetchone() == ( + 0, + ) + + +def test_compact_outbox_location_retains_valid_admitted_journal_pair(tmp_path): + from determa.state.checkpoint import seal_execution_checkpoint + from determa.state.checkpoint_v1 import restore_execution_checkpoint_v1 + from determa.state.host import outbox_intent_digest + + host, root, request, context = host_fixture(tmp_path) + host.submit_result(root, request, **context) + host.terminalize_outbox(root, request["effect_id"], {"status": "confirmed"}) + saved = host.snapshot(root) + checkpoint, journal = saved["checkpoint"], saved["journal"] + terminal = checkpoint["terminal_outbox_records"].pop() + checkpoint["outbox_effect_tombstones"].append( + { + "terminal_sequence": terminal["terminal_sequence"], + "effect_id": terminal["intent"]["effect_id"], + "intent_digest": outbox_intent_digest(root, terminal["intent"]), + "committed_revision": terminal["committed_revision"], + "outcome": terminal["outcome"], + } + ) + checkpoint["revision"] = str(int(checkpoint["revision"]) + 1) + checkpoint = seal_execution_checkpoint(checkpoint) + restore_execution_checkpoint_v1(checkpoint, host.resolver) + journal["checkpoint_revision"] = checkpoint["revision"] + journal["checkpoint_digest"] = checkpoint["execution_checkpoint_digest"] + journal["journal_revision"] = str(int(journal["journal_revision"]) + 1) + journal = seal_journal(journal) + validate_journal(checkpoint, journal) + restored = SQLiteCommittedEffectHost(tmp_path / "compact.sqlite", host.resolver, {}, None) + restored.setup_schema() + restored.seed(checkpoint, journal) + assert restored.recover(root)["journal"] == journal + + +@pytest.mark.parametrize("changed", ["payload", "token", "event", "target", "receipt_kind", "mode"]) +def test_admission_receipt_authenticates_exact_pinned_result(tmp_path, changed): + from determa.state.wire import hash_value + + host, root, request, context = host_fixture(tmp_path) + host.submit_result(root, request, **context) + saved = host.snapshot(root) + journal = saved["journal"] + record = journal["effect_records"][0] + outcome = record["outcome"] + if changed == "payload": + outcome["payload"][1][0][1][1] = "altered" + elif changed == "token": + record["operation_token"] = "altered" + elif changed == "event": + record["result_mapping"][0]["event"] = "native_cancelled" + elif changed == "target": + record["target"]["runtime_id"] = "different-runtime" + elif changed == "receipt_kind": + record["admission_receipt"]["operation_kind"] = "event_terminal" + else: + record["admission_receipt"]["delivery_mode"] = "signal" + outcome["digest"] = hash_value( + [ + "determa-effect-outcome-1", + record["effect_id"], + record["operation_token"], + outcome["kind"], + outcome["payload"], + outcome["attempt_fence"], + ] + ) + report = record["attempt_records"][0] + report["report_digest"] = hash_value( + [ + "determa-effect-attempt-report-1", + record["effect_id"], + record["operation_token"], + outcome["attempt_fence"], + outcome["kind"], + outcome["payload"], + report["reason"], + ] + ) + with pytest.raises(EffectError, match="invalid_effect_journal"): + validate_journal(saved["checkpoint"], seal_journal(journal), host.resolver) + + +def test_historical_result_evidence_survives_processing(tmp_path): + from determa.state.checkpoint_v1 import step_checkpoint_v1 + + host, root, request, context = host_fixture(tmp_path) + host.submit_result(root, request, **context) + saved = host.snapshot(root) + checkpoint = saved["checkpoint"] + checkpoint = step_checkpoint_v1( + checkpoint, + checkpoint["root_record"]["aggregate_state"]["root_runtime_id"], + host.resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + journal = saved["journal"] + journal["checkpoint_revision"] = checkpoint["revision"] + journal["checkpoint_digest"] = checkpoint["execution_checkpoint_digest"] + journal = seal_journal(journal) + validate_journal(checkpoint, journal, host.resolver) + restored = SQLiteCommittedEffectHost(tmp_path / "processed.sqlite", host.resolver, {}, None) + restored.setup_schema() + restored.seed(checkpoint, journal) + assert restored.recover(root)["journal"] == journal From abfa21b00cc84d64cf54e46938fe1fc579472ac7 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 10:41:15 +0000 Subject: [PATCH 26/30] fix: verify native destination receipts before ambiguous retry --- conformance/effects_adapter.py | 23 ++++- src/determa/state/effects.py | 129 +++++++++++++++++++++++++++- tests/test_committed_effects.py | 146 +++++++++++++++++++++++++++++++- 3 files changed, 290 insertions(+), 8 deletions(-) diff --git a/conformance/effects_adapter.py b/conformance/effects_adapter.py index 4d933b0..f599a7e 100644 --- a/conformance/effects_adapter.py +++ b/conformance/effects_adapter.py @@ -409,6 +409,11 @@ def health(self, instance: Any) -> str: def invoke(self, instance: Any, payload: Any, metadata: Any, attempt: Any) -> Any: return self.invocation(payload, metadata, attempt) + def verify_deduplication_evidence(self, instance: Any, evidence: Any) -> bool: + return instance["destination_binding_digest"] == _sha( + _destination_configuration() + ) and evidence == _destination_proof(evidence["scope_identity"], evidence["effect_id"]) + def _run_operation(payload: dict[str, Any], control: Path | None = None) -> dict[str, Any]: checkpoint, journal = payload["checkpoint_before"], payload["journal_before"] @@ -473,6 +478,10 @@ def factory() -> _FixtureNativeProvider: return provider factory_code, handler_code = factory.__code__, handler.__code__ + destination_codes = { + name: globals()[name].__code__ + for name in ("_destination_proof", "_destination_configuration", "_destination_call") + } installed_invoke, installed_reply = handler_module.invoke, handler_module.NativeReply installed_members = { name: member @@ -499,6 +508,7 @@ def verify_native(source: Any, selected: Any) -> bool: and type(provider) is _FixtureNativeProvider and provider.invocation is handler and handler.__code__ is handler_code + and all(globals()[name].__code__ is code for name, code in destination_codes.items()) and all( getattr(_FixtureNativeProvider, name).__code__ is code for name, code in provider_methods.items() @@ -631,6 +641,17 @@ def before_commit(candidate: dict[str, Any], proposed_journal: dict[str, Any]) - caller_kind = "aborted" elif operation == "claim": try: + evidence = None + if configuration["destination_deduplication_proven"] and any( + record["effect_id"] == arguments["effect_id"] + and record["invocation_state"] == "ambiguous" + for record in journal["effect_records"] + ): + # Exercise the actual installed destination twice. The configuration + # requests this probe; only native receipt verification permits retry. + _destination_call(_AUTHORITY_SCOPE, arguments["effect_id"], None) + _destination_call(_AUTHORITY_SCOPE, arguments["effect_id"], None) + evidence = _destination_proof(_AUTHORITY_SCOPE, arguments["effect_id"]) claim = host.claim( _ROOT, arguments["effect_id"], @@ -638,7 +659,7 @@ def before_commit(candidate: dict[str, Any], proposed_journal: dict[str, Any]) - context["scope_authority_epoch"], expires_at=str(int(context["trusted_host_now"]) + 1), trusted_now=context["trusted_host_now"], - deduplication_proven=configuration["destination_deduplication_proven"], + deduplication_evidence=evidence, ) new_claims.append( { diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index c3bbae9..635560b 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -7,6 +7,7 @@ from __future__ import annotations +import base64 import copy import json import re @@ -182,6 +183,54 @@ def _pinned_result_target( ) -> dict[str, Any]: target = record["target"] origin = target["runtime_incarnation"] + root, runtime_id = checkpoint["root_instance_id"], target["runtime_id"] + if target["root_instance_id"] != root: + raise EffectError("invalid_effect_journal") + definition = origin["definition"] + machine = definition["machine"] + identity = [machine["namespace"], machine["machine_id"], machine["machine_version"]] + if origin["kind"] == "root": + expected_id = hash_value( + [ + "determa-root-runtime-identity-1", + "1", + definition["validated_bundle_fingerprint"], + *identity, + origin["root_instance_id"], + ] + ) + elif origin["kind"] == "component": + if origin["declaration_index"] != origin["component_definition_pointer"].rsplit("/", 1)[-1]: + raise EffectError("invalid_effect_journal") + expected_id = hash_value( + [ + "determa-component-runtime-identity-1", + "1", + root, + origin["owner_runtime_id"], + origin["component_definition_pointer"], + origin["activation_sequence"], + *identity, + ] + ) + else: + expected_id = hash_value( + [ + "determa-spawned-runtime-identity-1", + "1", + root, + origin["owner_runtime_id"], + origin["spawn_action_pointer"], + origin["spawn_sequence"], + *identity, + ] + ) + if expected_id != runtime_id: + raise EffectError("invalid_effect_journal") + if resolver is not None: + from .wire import _origin_machine + + _origin_machine(resolver, origin) aggregate = checkpoint["root_record"].get("aggregate_state") for runtime in aggregate["runtimes"] if aggregate else []: if runtime["runtime_id"] == target["runtime_id"] and runtime["identity_origin"] == origin: @@ -646,6 +695,29 @@ def verify(self, reference: Mapping[str, Any], destination: str) -> tuple[Any, A except (ExtensionError, ValueError, TypeError, KeyError) as error: raise EffectError("host_capability_mismatch") from error + def verify_deduplication_evidence( + self, + reference: Mapping[str, Any], + destination: str, + evidence: Mapping[str, Any], + ) -> None: + """Verify actual destination evidence through the installed native provider. + + The provider must independently authenticate receipts against its native + destination, rather than trust caller-supplied equal bytes. + """ + from .runtime_providers import _bound_provider_method + + VerifiedNativeHandler.verify(self, reference, destination) + try: + _descriptor, provider, instance, _evaluator = self._registry._bound(self._configured) + method = _bound_provider_method(provider, "verify_deduplication_evidence") + if method(instance, copy.deepcopy(dict(evidence))) is not True: + raise EffectError("host_capability_mismatch") + VerifiedNativeHandler.verify(self, reference, destination) + except (ExtensionError, ValueError, TypeError, KeyError, AttributeError) as exc: + raise EffectError("host_capability_mismatch") from exc + def invoke( self, reference: Mapping[str, Any], @@ -1145,18 +1217,69 @@ def claim( *, expires_at: str, trusted_now: str, - deduplication_proven: bool = False, + deduplication_evidence: Mapping[str, Any] | None = None, ) -> dict[str, Any]: def change(document: dict[str, Any]) -> dict[str, Any]: - return _issue_effect_claim( + record = _record(document["journal"], effect_id) + proof = None + if deduplication_evidence is not None: + if not isinstance(deduplication_evidence, Mapping): + raise EffectError("host_capability_mismatch") + proof = copy.deepcopy(dict(deduplication_evidence)) + fields = { + "scope_identity", + "effect_id", + "destination_binding_digest", + "first_attempt_receipt_bytes_base64", + "repeat_attempt_receipt_bytes_base64", + } + if ( + set(proof) != fields + or proof["scope_identity"] != document["journal"]["scope_identity"] + or proof["effect_id"] != effect_id + or proof["destination_binding_digest"] != record["destination_binding_digest"] + ): + raise EffectError("host_capability_mismatch") + try: + first = base64.b64decode( + proof["first_attempt_receipt_bytes_base64"], validate=True + ) + repeated = base64.b64decode( + proof["repeat_attempt_receipt_bytes_base64"], validate=True + ) + if not first or first != repeated: + raise ValueError("destination receipts differ") + except (ValueError, TypeError) as exc: + raise EffectError("host_capability_mismatch") from exc + handler = self._verified_handler( + record["handler_reference"], record["destination_binding_digest"] + ) + VerifiedNativeHandler.verify_deduplication_evidence( + handler, + record["handler_reference"], + record["destination_binding_digest"], + proof, + ) + claim = _issue_effect_claim( document, effect_id, principal, epoch, expires_at, trusted_now, - deduplication_proven=deduplication_proven, + deduplication_proven=proof is not None, ) + if proof is not None: + document.setdefault("destination_evidence", {}).setdefault(effect_id, []).append( + { + "root_instance_id": root, + "operation_token": record["operation_token"], + "handler_reference": copy.deepcopy(record["handler_reference"]), + "attempt_fence": claim["attempt_fence"], + "evidence": proof, + } + ) + return claim return self._transact( root, change, expected_epoch=epoch, authority_mutation=self._mirror_authority diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index bfb2138..cd41e59 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -22,8 +22,9 @@ def read(name): class NativeTestProvider: - def __init__(self, callback): + def __init__(self, callback, proof_verifier=None): self.callback = callback + self.proof_verifier = proof_verifier def validate_configuration(self, configuration): return copy.deepcopy(configuration) @@ -37,16 +38,20 @@ def health(self, instance): def invoke(self, instance, payload, metadata, attempt): return self.callback(payload, metadata, attempt) + def verify_deduplication_evidence(self, instance, evidence): + return self.proof_verifier is not None and self.proof_verifier(evidence) -def installed_test_handler(host, callback): + +def installed_test_handler(host, callback, proof_verifier=None): from determa.state.effects import VerifiedNativeHandler from determa.state.extensions import ExtensionRegistry - provider = NativeTestProvider(callback) + provider = NativeTestProvider(callback, proof_verifier) def factory(): return provider + proof_code = proof_verifier.__code__ if proof_verifier is not None else None callback_code = callback.__code__ factory_code = factory.__code__ methods = { @@ -68,6 +73,8 @@ def verify(source, actual): and factory.__code__ is factory_code and type(provider) is NativeTestProvider and provider.callback is callback + and provider.proof_verifier is proof_verifier + and (proof_verifier is None or proof_verifier.__code__ is proof_code) and callback.__code__ is callback_code and all( getattr(NativeTestProvider, name).__code__ is code for name, code in methods.items() @@ -991,7 +998,10 @@ def test_compact_outbox_location_retains_valid_admitted_journal_pair(tmp_path): assert restored.recover(root)["journal"] == journal -@pytest.mark.parametrize("changed", ["payload", "token", "event", "target", "receipt_kind", "mode"]) +@pytest.mark.parametrize( + "changed", + ["payload", "token", "event", "target", "target_root", "origin", "receipt_kind", "mode"], +) def test_admission_receipt_authenticates_exact_pinned_result(tmp_path, changed): from determa.state.wire import hash_value @@ -1009,6 +1019,12 @@ def test_admission_receipt_authenticates_exact_pinned_result(tmp_path, changed): record["result_mapping"][0]["event"] = "native_cancelled" elif changed == "target": record["target"]["runtime_id"] = "different-runtime" + elif changed == "target_root": + record["target"]["root_instance_id"] = "foreign-root" + elif changed == "origin": + record["target"]["runtime_incarnation"]["definition"]["validated_bundle_fingerprint"] = ( + "sha256:" + "0" * 64 + ) elif changed == "receipt_kind": record["admission_receipt"]["operation_kind"] = "event_terminal" else: @@ -1062,3 +1078,125 @@ def test_historical_result_evidence_survives_processing(tmp_path): restored.setup_schema() restored.seed(checkpoint, journal) assert restored.recover(root)["journal"] == journal + + +@pytest.mark.parametrize( + "damage", + [None, "boolean", "fabricated", "wrong_work", "wrong_destination", "no_native_receipt"], +) +def test_ambiguous_retry_requires_verified_native_destination_evidence(tmp_path, damage): + import base64 + import sqlite3 + + original, root, request, _ = host_fixture(tmp_path) + journal = read("data/ambiguous-journal.json") + host = SQLiteCommittedEffectHost(tmp_path / "ambiguous.sqlite", original.resolver, {}, None) + host.setup_schema() + host.seed(read("pending-checkpoint.json"), journal) + record = journal["effect_records"][0] + host.route.update( + {key: record[key] for key in ("handler_reference", "destination_binding_digest")} + ) + native_path = tmp_path / "destination.sqlite" + receipt_bytes = b"actual native receipt for scoped effect" + with sqlite3.connect(native_path) as connection: + connection.execute( + "CREATE TABLE receipts (scope TEXT, effect TEXT, receipt BLOB, calls INTEGER)" + ) + if damage != "no_native_receipt": + connection.execute( + "INSERT INTO receipts VALUES (?, ?, ?, 2)", + ( + journal["scope_identity"], + record["effect_id"], + receipt_bytes, + ), + ) + + def verify_receipts(evidence): + with sqlite3.connect(native_path) as connection: + row = connection.execute( + "SELECT receipt, calls FROM receipts WHERE scope = ? AND effect = ?", + ( + evidence["scope_identity"], + evidence["effect_id"], + ), + ).fetchone() + return ( + row is not None + and row[1] >= 2 + and all( + base64.b64decode(evidence[name], validate=True) == row[0] + for name in ( + "first_attempt_receipt_bytes_base64", + "repeat_attempt_receipt_bytes_base64", + ) + ) + ) + + host = installed_test_handler(host, lambda *_: {}, verify_receipts) + encoded = base64.b64encode(receipt_bytes).decode() + proof = { + "scope_identity": journal["scope_identity"], + "effect_id": record["effect_id"], + "destination_binding_digest": record["destination_binding_digest"], + "first_attempt_receipt_bytes_base64": encoded, + "repeat_attempt_receipt_bytes_base64": encoded, + } + if damage == "boolean": + proof = True + elif damage == "fabricated": + proof["first_attempt_receipt_bytes_base64"] = proof[ + "repeat_attempt_receipt_bytes_base64" + ] = base64.b64encode(b"fabricated equal bytes").decode() + elif damage == "wrong_work": + proof["effect_id"] = "sha256:" + "0" * 64 + elif damage == "wrong_destination": + proof["destination_binding_digest"] = "sha256:" + "0" * 64 + before = host.snapshot(root) + if damage is not None: + with pytest.raises(EffectError, match="host_capability_mismatch"): + host.claim( + root, + request["effect_id"], + "worker", + "0", + expires_at="10", + trusted_now="0", + deduplication_evidence=proof, + ) + assert host.snapshot(root) == before + else: + claim = host.claim( + root, + request["effect_id"], + "worker", + "0", + expires_at="10", + trusted_now="0", + deduplication_evidence=proof, + ) + assert int(claim["attempt_fence"]) == int(record["attempt_fence"]) + 1 + saved = host.snapshot(root)["destination_evidence"][record["effect_id"]][0] + assert saved["evidence"] == proof + assert saved["root_instance_id"] == root + assert saved["attempt_fence"] == claim["attempt_fence"] + reopened = SQLiteCommittedEffectHost(host.path, host.resolver, {}, None) + assert ( + reopened.snapshot(root)["destination_evidence"] + == host.snapshot(root)["destination_evidence"] + ) + + +def test_ambiguous_retry_does_not_accept_legacy_boolean_keyword(tmp_path): + host, root, request, _ = host_fixture(tmp_path) + with pytest.raises(TypeError, match="deduplication_proven"): + host.claim( + root, + request["effect_id"], + "worker", + "0", + expires_at="10", + trusted_now="0", + deduplication_proven=True, + ) From 1213be5482a0608b69c52a44929a55264e929898 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 10:43:18 +0000 Subject: [PATCH 27/30] test: retain native result identity after terminal recovery --- README.md | 14 +- src/determa/state/effects.py | 2 +- tests/test_committed_effects.py | 220 +++++++++++++++++++++++++++++++- 3 files changed, 231 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b1fb4e7..4dcc399 100644 --- a/README.md +++ b/README.md @@ -416,9 +416,17 @@ independently authenticated principal, current scope/epoch/attempt claim and an explicit `trusted_clock` callback. Clock failures or expiry fail closed, including expiry during outcome recording before its native commit. An expired external attempt remains unresolved and requires destination idempotency or reconciliation; -lease expiry does not prove the call was undone. - -Authenticated outcome recording commits before result-event admission. A crash +lease expiry does not prove the call was undone. An ambiguous retry requires +`deduplication_evidence` bound to the scope, effect and destination. The installed +provider's `verify_deduplication_evidence(instance, evidence)` must authenticate +both receipt byte strings against its actual native destination; equal bytes from +a caller are insufficient. The host retains the verified evidence with the new +attempt fence in the same transaction as the claim. + +The public `submit_result` returns `committed` only after result-event admission. +Recovery verifies the exact checkpoint acceptance receipt against the pinned +payload, token mapping and original target identity, even after processing or +root tombstoning. Authenticated outcome recording commits before result-event admission. A crash between those transactions retains the outcome, and `recover` admits it without calling the provider again. Each recoverable effect commits separately. Duplicate results retain exact first responses; conflicting or stale work is refused. diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 635560b..8ca6bf2 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -715,7 +715,7 @@ def verify_deduplication_evidence( if method(instance, copy.deepcopy(dict(evidence))) is not True: raise EffectError("host_capability_mismatch") VerifiedNativeHandler.verify(self, reference, destination) - except (ExtensionError, ValueError, TypeError, KeyError, AttributeError) as exc: + except Exception as exc: raise EffectError("host_capability_mismatch") from exc def invoke( diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index cd41e59..4ad0108 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -1082,7 +1082,15 @@ def test_historical_result_evidence_survives_processing(tmp_path): @pytest.mark.parametrize( "damage", - [None, "boolean", "fabricated", "wrong_work", "wrong_destination", "no_native_receipt"], + [ + None, + "boolean", + "fabricated", + "wrong_work", + "wrong_destination", + "no_native_receipt", + "unavailable", + ], ) def test_ambiguous_retry_requires_verified_native_destination_evidence(tmp_path, damage): import base64 @@ -1114,6 +1122,8 @@ def test_ambiguous_retry_requires_verified_native_destination_evidence(tmp_path, ) def verify_receipts(evidence): + if damage == "unavailable": + raise RuntimeError("destination proof unavailable") with sqlite3.connect(native_path) as connection: row = connection.execute( "SELECT receipt, calls FROM receipts WHERE scope = ? AND effect = ?", @@ -1200,3 +1210,211 @@ def test_ambiguous_retry_does_not_accept_legacy_boolean_keyword(tmp_path): trusted_now="0", deduplication_proven=True, ) + + +def test_historical_admission_survives_actual_root_completion_and_tombstone(tmp_path): + import yaml + + from determa.state import ExecutionHost, MemoryExecutionStore, create_checkpoint_v1 + from determa.state.checkpoint_v1 import admit_checkpoint_v1, step_checkpoint_v1 + from determa.state.host import outbox_intent_digest + from determa.state.queueing import _entry_digest + from determa.state.wire import canonical_bytes + + machine = yaml.safe_load((CASE / "machine.yaml").read_text()) + events = machine["machines"][0]["root"]["on_events"] + events["native_succeeded"] = {"transition_to": "done"} + machine["machines"][0]["root"] = { + "type": "composite", + "initial": {"transition_to": "working"}, + "states": {"working": {"on_events": events}, "done": {"type": "final"}}, + } + bundle = load_bundle(machine) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + root = "historical-root" + checkpoint = create_checkpoint_v1(bundle, "workflow", root, "creation") + runtime = checkpoint["root_record"]["aggregate_state"]["runtimes"][0] + envelope = { + "event": "invoke", + "event_id": "invoke", + "cause_id": "invoke", + "source": {"host": True}, + "target": runtime["target_identity"], + "payload": ["map", [["operation_token", ["string", "business-order-42"]]]], + } + checkpoint = admit_checkpoint_v1( + checkpoint, + [ + { + "delivery_mode": "input", + "envelope": envelope, + "envelope_digest": _entry_digest(root, "input", envelope), + } + ], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + checkpoint = step_checkpoint_v1( + checkpoint, + runtime["runtime_id"], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + journal = read("data/unclaimed-journal.json") + journal.update( + root_instance_id=root, + checkpoint_revision=checkpoint["revision"], + checkpoint_digest=checkpoint["execution_checkpoint_digest"], + operation_response_references=[], + ) + record = journal["effect_records"][0] + intent = checkpoint["pending_outbox_intents"][0]["intent"] + record.update( + effect_id=intent["effect_id"], + intent_digest=outbox_intent_digest(root, intent), + target={ + "root_instance_id": root, + "runtime_id": runtime["runtime_id"], + "runtime_incarnation": runtime["identity_origin"], + }, + ) + host = SQLiteCommittedEffectHost( + tmp_path / "terminal.sqlite", resolver, {}, None, trusted_clock=lambda: "0" + ) + host.setup_schema() + host.seed(checkpoint, seal_journal(journal)) + host.claim(root, record["effect_id"], "worker", "0", expires_at="10", trusted_now="0") + request = read("data/result-request.json") + request["effect_id"] = record["effect_id"] + host.submit_result( + root, + request, + principal="worker", + scope=journal["scope_identity"], + epoch="0", + trusted_now="0", + ) + host.terminalize_outbox(root, record["effect_id"], {"status": "confirmed"}) + saved = host.snapshot(root) + checkpoint = saved["checkpoint"] + checkpoint = step_checkpoint_v1( + checkpoint, + runtime["runtime_id"], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + assert checkpoint["root_record"]["aggregate_state"]["runtimes"][0]["status"] == "completed" + portable_host = ExecutionHost( + MemoryExecutionStore({root: canonical_bytes(checkpoint)}), resolver + ) + portable_host.tombstone_root_v1( + root, + "tombstone", + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + checkpoint = portable_host.read_checkpoint(root).document + journal = saved["journal"] + journal["checkpoint_revision"] = checkpoint["revision"] + journal["checkpoint_digest"] = checkpoint["execution_checkpoint_digest"] + journal = seal_journal(journal) + restored = SQLiteCommittedEffectHost(tmp_path / "tombstoned.sqlite", resolver, {}, None) + restored.setup_schema() + restored.seed(checkpoint, journal) + assert restored.recover(root)["journal"] == journal + + +@pytest.mark.parametrize( + "case, machine_id, kind", + [ + ("54-stale-component-target", "owner", "component"), + ("30-owned-spawn-cancel", "order", "owned_spawned_instance"), + ], +) +def test_historical_child_target_uses_original_definition_after_removal(case, machine_id, kind): + from determa.state import create_checkpoint_v1 + from determa.state.checkpoint_v1 import admit_checkpoint_v1, step_checkpoint_v1 + from determa.state.effects import _pinned_result_target + from determa.state.queueing import _entry_digest + + bundle = load_bundle( + (conformance_root() / "conformance/core" / case / "machine.yaml").read_text() + ) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + checkpoint = create_checkpoint_v1(bundle, machine_id, "root", "creation") + runtime = checkpoint["root_record"]["aggregate_state"]["runtimes"][0] + envelope = { + "event": "start", + "event_id": "start", + "cause_id": "start", + "source": {"host": True}, + "target": runtime["target_identity"], + "payload": ["map", []], + } + checkpoint = admit_checkpoint_v1( + checkpoint, + [ + { + "delivery_mode": "input", + "envelope": envelope, + "envelope_digest": _entry_digest("root", "input", envelope), + } + ], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + checkpoint = step_checkpoint_v1( + checkpoint, + runtime["runtime_id"], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + child = next( + item + for item in checkpoint["root_record"]["aggregate_state"]["runtimes"] + if item["identity_origin"]["kind"] == kind + ) + record = { + "target": { + "root_instance_id": "root", + "runtime_id": child["runtime_id"], + "runtime_incarnation": copy.deepcopy(child["identity_origin"]), + } + } + event = "leave" if kind == "component" else "cancel_payment" + envelope.update(event=event, event_id=event, cause_id=event) + checkpoint = admit_checkpoint_v1( + checkpoint, + [ + { + "delivery_mode": "input", + "envelope": envelope, + "envelope_digest": _entry_digest("root", "input", envelope), + } + ], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + checkpoint = step_checkpoint_v1( + checkpoint, + runtime["runtime_id"], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + assert all( + item["runtime_id"] != child["runtime_id"] + for item in checkpoint["root_record"]["aggregate_state"]["runtimes"] + ) + assert _pinned_result_target(checkpoint, record, resolver) == child["target_identity"] + origin = record["target"]["runtime_incarnation"] + counter = "activation_sequence" if kind == "component" else "spawn_sequence" + origin[counter] = str(int(origin[counter]) + 1) + with pytest.raises(EffectError, match="invalid_effect_journal"): + _pinned_result_target(checkpoint, record, resolver) From 60e0464855eb5015ebaf8d897c94b93df8b2cfa6 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 10:48:57 +0000 Subject: [PATCH 28/30] fix: bind effect producer replay to original operation identity --- src/determa/state/effects.py | 45 +++++++++++++- tests/test_committed_effects.py | 100 ++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 2 deletions(-) diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 8ca6bf2..f744e2c 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -492,7 +492,10 @@ def _retained_report( def _reconstruct_producer_response( - checkpoint: Mapping[str, Any], journal: Mapping[str, Any], operation_id: str + checkpoint: Mapping[str, Any], + journal: Mapping[str, Any], + operation_id: str, + request_identity: Mapping[str, Any], ) -> dict[str, Any] | None: """Resolve a retained simple producer response from complete checkpoint evidence.""" reference = next( @@ -514,6 +517,23 @@ def _reconstruct_producer_response( if len(receipts) != 1: raise EffectError("replay_evidence_expired") receipt = receipts[0] + emission_ids = { + item["effect_id"] + for item in receipt["emission_references"] + if item["kind"] == "external_outbox" + } + records = [ + record for record in journal["effect_records"] if record["effect_id"] in emission_ids + ] + if len(records) != len(emission_ids) or not records: + raise EffectError("replay_evidence_expired") + if any( + record["target"]["root_instance_id"] != request_identity["root_instance_id"] + or record["target"]["runtime_id"] != request_identity["target_runtime_id"] + or record["operation_token"] != request_identity["operation_token"] + for record in records + ): + raise EffectError("operation_id_conflict") intents = { item["intent"]["effect_id"]: item["intent"] for item in checkpoint["pending_outbox_intents"] } @@ -802,6 +822,7 @@ def seed( "result_requests": {}, "result_responses": {}, "cancel_requests": {}, + "producer_requests": {}, } if claim is not None: document["claims"][claim["work_identity"]] = copy.deepcopy(dict(claim)) @@ -1108,12 +1129,29 @@ def produce( ) -> dict[str, Any]: """Step one selected event, pin every emitted external intent, then commit.""" + # CAS and route generation guard new work. Replay identifies the original + # production operation independently of current admission preconditions. + request_identity = { + "operation_kind": "produce", + "root_instance_id": root, + "target_runtime_id": target_runtime_id, + "operation_token": operation_token, + } + def change(document: dict[str, Any]) -> dict[str, Any]: saved = document["responses"].get(operation_id) if saved is not None: + if ( + document.get("producer_requests", {}).get(operation_id) != request_identity + or saved.get("kind") != "processing" + ): + raise EffectError("operation_id_conflict") return copy.deepcopy(saved) reconstructed = _reconstruct_producer_response( - document["checkpoint"], document["journal"], operation_id + document["checkpoint"], + document["journal"], + operation_id, + request_identity, ) if reconstructed is not None: return reconstructed @@ -1190,6 +1228,9 @@ def change(document: dict[str, Any]) -> dict[str, Any]: key=lambda item: item["operation_id"].encode() ) document["responses"][operation_id] = copy.deepcopy(response) + document.setdefault("producer_requests", {})[operation_id] = copy.deepcopy( + request_identity + ) document["checkpoint"] = candidate _bump(journal, candidate) if before_commit is not None: diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 4ad0108..a44c1f0 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -1418,3 +1418,103 @@ def test_historical_child_target_uses_original_definition_after_removal(case, ma origin[counter] = str(int(origin[counter]) + 1) with pytest.raises(EffectError, match="invalid_effect_journal"): _pinned_result_target(checkpoint, record, resolver) + + +def test_producer_replay_cannot_return_a_cancellation_response(tmp_path): + host, root, _, _ = host_fixture(tmp_path) + request = read("data/cancel-request.json") + request["operation_id"] = "shared-id" + host.cancel(root, request) + before = host.snapshot(root) + with pytest.raises(EffectError, match="operation_id_conflict"): + host.produce( + root, + "shared-id", + "different-runtime", + expected_revision="999", + expected_digest="bad", + route_generation="999", + operation_token="different-token", + ) + assert host.snapshot(root) == before + + +@pytest.mark.parametrize("damage", [None, "target", "token"]) +def test_portable_producer_replay_requires_original_work_identity(tmp_path, damage): + host, root, _, _ = host_fixture(tmp_path) + record = host.snapshot(root)["journal"]["effect_records"][0] + # Restoration has the normative response hash and immutable intent evidence, + # but no process-local producer response cache or current route. + target = record["target"]["runtime_id"] if damage != "target" else "different-runtime" + token = record["operation_token"] if damage != "token" else "different-token" + before = host.snapshot(root) + arguments = { + "expected_revision": "stale", + "expected_digest": "stale", + "route_generation": "999", + "operation_token": token, + } + if damage is not None: + with pytest.raises(EffectError, match="operation_id_conflict"): + host.produce(root, "produce-1", target, **arguments) + else: + # Fixture journal response hash describes its pre-claim producer response. + # Use the matching unclaimed journal for exact reconstruction. + restored = SQLiteCommittedEffectHost(tmp_path / "producer.sqlite", host.resolver, {}, None) + restored.setup_schema() + restored.seed(read("pending-checkpoint.json"), read("data/unclaimed-journal.json")) + assert restored.produce(root, "produce-1", target, **arguments) == read( + "data/producer-response.json" + ) + assert host.snapshot(root) == before + + +@pytest.mark.parametrize("damage", [None, "target", "token", "cancel"]) +def test_native_producer_replay_retains_exact_kind_and_request_identity(tmp_path, damage): + original, root, _, _ = host_fixture(tmp_path) + checkpoint = read("accepted-checkpoint.json") + journal = read("data/empty-journal.json") + record = read("data/unclaimed-journal.json")["effect_records"][0] + route = { + key: record[key] + for key in ( + "handler_reference", + "destination_binding_digest", + "result_mapping", + "target", + "idempotency_policy", + ) + } + route["generation"] = record["route_configuration_generation"] + host = SQLiteCommittedEffectHost(tmp_path / "production.sqlite", original.resolver, route, None) + host.setup_schema() + host.seed(checkpoint, journal) + host = installed_test_handler(host, lambda *_: {}) + runtime = checkpoint["root_record"]["aggregate_state"]["root_runtime_id"] + arguments = { + "expected_revision": checkpoint["revision"], + "expected_digest": checkpoint["execution_checkpoint_digest"], + "route_generation": route["generation"], + "operation_token": record["operation_token"], + } + first = host.produce(root, "shared-id", runtime, **arguments) + before = host.snapshot(root) + if damage == "cancel": + request = read("data/cancel-request.json") + request["operation_id"] = "shared-id" + with pytest.raises(EffectError, match="operation_id_conflict"): + host.cancel(root, request) + else: + host.route["generation"] = "999" + host.handler = None + arguments.update(expected_revision="stale", expected_digest="stale", route_generation="999") + if damage == "target": + runtime = "different-runtime" + elif damage == "token": + arguments["operation_token"] = "different-token" + if damage is not None: + with pytest.raises(EffectError, match="operation_id_conflict"): + host.produce(root, "shared-id", runtime, **arguments) + else: + assert host.produce(root, "shared-id", runtime, **arguments) == first + assert host.snapshot(root) == before From 293a7a48052ea004db8686843e12389c4ebab638 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 10:53:15 +0000 Subject: [PATCH 29/30] fix: derive restored producer identity from committed effect IDs --- conformance/effects_adapter.py | 6 ++- src/determa/state/effects.py | 65 ++++++++++++++++++++++++++++++--- tests/test_committed_effects.py | 30 ++++++++++++++- 3 files changed, 92 insertions(+), 9 deletions(-) diff --git a/conformance/effects_adapter.py b/conformance/effects_adapter.py index f599a7e..9b59466 100644 --- a/conformance/effects_adapter.py +++ b/conformance/effects_adapter.py @@ -694,14 +694,16 @@ def before_commit(candidate: dict[str, Any], proposed_journal: dict[str, Any]) - elif operation == "terminalize_outbox": host.terminalize_outbox(_ROOT, arguments["effect_id"], {"status": arguments["status"]}) elif operation == "submit_result": - response = host.submit_result( + submit = ( + host._record_result if fault == "after_outcome_before_admission" else host.submit_result + ) + response = submit( _ROOT, arguments, principal=context["principal"], scope=context["scope_identity"], epoch=context["scope_authority_epoch"], trusted_now=context["trusted_host_now"], - admit=fault != "after_outcome_before_admission", ) caller_kind = "response" if fault == "after_outcome_before_admission": diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index f744e2c..7cc8613 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -496,6 +496,7 @@ def _reconstruct_producer_response( journal: Mapping[str, Any], operation_id: str, request_identity: Mapping[str, Any], + resolver: ArtifactResolver, ) -> dict[str, Any] | None: """Resolve a retained simple producer response from complete checkpoint evidence.""" reference = next( @@ -527,12 +528,7 @@ def _reconstruct_producer_response( ] if len(records) != len(emission_ids) or not records: raise EffectError("replay_evidence_expired") - if any( - record["target"]["root_instance_id"] != request_identity["root_instance_id"] - or record["target"]["runtime_id"] != request_identity["target_runtime_id"] - or record["operation_token"] != request_identity["operation_token"] - for record in records - ): + if any(record["operation_token"] != request_identity["operation_token"] for record in records): raise EffectError("operation_id_conflict") intents = { item["intent"]["effect_id"]: item["intent"] for item in checkpoint["pending_outbox_intents"] @@ -550,6 +546,62 @@ def _reconstruct_producer_response( ] if len(emissions) != len(receipt["emission_references"]): raise EffectError("replay_evidence_expired") + # Result routing can name a different runtime from the emitting producer. + # Derive producer identity from actual effect IDs and trusted author send + # locations. Complex lifecycle emission histories without this evidence + # remain unavailable for portable reconstruction. + from .engine import _pointer_get + from .wire import _origin_machine + + try: + aggregate = checkpoint["root_record"]["aggregate_state"] + source_runtime = next( + item + for item in aggregate["runtimes"] + if item["runtime_id"] == request_identity["target_runtime_id"] + ) + bundle, machine = _origin_machine( + resolver, {"definition": source_runtime["current_definition"]} + ) + namespace, machine_id, version = machine.definition_identity() + root_pointer = source_runtime["current_definition"]["machine"]["root_definition_pointer"] + pointers: list[str] = [] + + def visit(value: Any, pointer: str) -> None: + if isinstance(value, dict): + if "send" in value: + pointers.append(pointer + "/send") + for name, child in value.items(): + visit(child, pointer + "/" + name.replace("~", "~0").replace("/", "~1")) + elif isinstance(value, list): + for index, child in enumerate(value): + visit(child, pointer + "/" + str(index)) + + visit(_pointer_get(bundle.raw, root_pointer), root_pointer) + step_sequence = str(int(aggregate["next_logical_step_sequence"]) - 1) + for emission in receipt["emission_references"]: + if emission["kind"] != "external_outbox": + continue + if not any( + hash_value( + [ + "determa-effect-identity-1", + "1", + [namespace, machine_id, str(version)], + request_identity["root_instance_id"], + request_identity["target_runtime_id"], + receipt["event_id"], + step_sequence, + pointer, + emission["emission_index"], + ] + ) + == emission["effect_id"] + for pointer in pointers + ): + raise EffectError("replay_evidence_expired") + except (ArtifactError, KeyError, TypeError, ValueError, StopIteration) as exc: + raise EffectError("replay_evidence_expired") from exc outcome = receipt["outcome"] response = { "kind": "processing", @@ -1152,6 +1204,7 @@ def change(document: dict[str, Any]) -> dict[str, Any]: document["journal"], operation_id, request_identity, + self.resolver, ) if reconstructed is not None: return reconstructed diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index a44c1f0..78189cf 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -1455,7 +1455,10 @@ def test_portable_producer_replay_requires_original_work_identity(tmp_path, dama "operation_token": token, } if damage is not None: - with pytest.raises(EffectError, match="operation_id_conflict"): + with pytest.raises( + EffectError, + match="replay_evidence_expired" if damage == "target" else "operation_id_conflict", + ): host.produce(root, "produce-1", target, **arguments) else: # Fixture journal response hash describes its pre-claim producer response. @@ -1518,3 +1521,28 @@ def test_native_producer_replay_retains_exact_kind_and_request_identity(tmp_path else: assert host.produce(root, "shared-id", runtime, **arguments) == first assert host.snapshot(root) == before + + +def test_portable_producer_identity_is_not_inferred_from_result_destination(tmp_path): + original, root, _, _ = host_fixture(tmp_path) + checkpoint = read("pending-checkpoint.json") + journal = read("data/unclaimed-journal.json") + record = journal["effect_records"][0] + source_runtime = checkpoint["root_record"]["aggregate_state"]["root_runtime_id"] + record["target"]["runtime_id"] = "another-result-runtime" + host = SQLiteCommittedEffectHost( + tmp_path / "different-target.sqlite", original.resolver, {}, None + ) + host.setup_schema() + host.seed(checkpoint, seal_journal(journal)) + arguments = { + "expected_revision": "stale", + "expected_digest": "stale", + "route_generation": "999", + "operation_token": record["operation_token"], + } + assert host.produce(root, "produce-1", source_runtime, **arguments) == read( + "data/producer-response.json" + ) + with pytest.raises(EffectError, match="replay_evidence_expired"): + host.produce(root, "produce-1", record["target"]["runtime_id"], **arguments) From d72c62ca3c75f0acbacde32f9ab1fe03cd7b633c Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Mon, 5 Oct 2026 10:55:38 +0000 Subject: [PATCH 30/30] fix: refuse compact producer replay when intent evidence expired --- src/determa/state/effects.py | 2 ++ tests/test_committed_effects.py | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/determa/state/effects.py b/src/determa/state/effects.py index 7cc8613..9691ce7 100644 --- a/src/determa/state/effects.py +++ b/src/determa/state/effects.py @@ -539,6 +539,8 @@ def _reconstruct_producer_response( for item in checkpoint["terminal_outbox_records"] } ) + if not emission_ids.issubset(intents): + raise EffectError("replay_evidence_expired") emissions = [ copy.deepcopy(intents[item["effect_id"]]) for item in receipt["emission_references"] diff --git a/tests/test_committed_effects.py b/tests/test_committed_effects.py index 78189cf..6b26f25 100644 --- a/tests/test_committed_effects.py +++ b/tests/test_committed_effects.py @@ -996,6 +996,19 @@ def test_compact_outbox_location_retains_valid_admitted_journal_pair(tmp_path): restored.setup_schema() restored.seed(checkpoint, journal) assert restored.recover(root)["journal"] == journal + before = restored.snapshot(root) + record = journal["effect_records"][0] + with pytest.raises(EffectError, match="replay_evidence_expired"): + restored.produce( + root, + "produce-1", + checkpoint["root_record"]["aggregate_state"]["root_runtime_id"], + expected_revision="stale", + expected_digest="stale", + route_generation="999", + operation_token=record["operation_token"], + ) + assert restored.snapshot(root) == before @pytest.mark.parametrize(