diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 07803ffca9..fb511f7c41 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -23,7 +23,7 @@ {"_type":"issue","id":"polylogue-cnaj","title":"Bound active JSONL append ingestion memory and catch-up overlap","description":"Live v35 incident on 2026-07-13: an actively appended 46 MB Codex JSONL was selected by periodic catch-up every ~16 seconds. Each append reported 0.1–0.3 MB read but held daemon writer 37–38 seconds and temporarily grew anonymous RSS from ~0.4 GiB to ~4.2 GiB; cgroup memory reached the 8 GiB high threshold (6,655 high events), 22 GB reads and 3.2 GB writes in 8.5 minutes. Daemon was intentionally stopped before OOM. This blocks safe unattended backfill/daemon operation.","design":"Build a reproducible harness from the observed active-append shape, then locate retained full-session/materialization state and overlapping periodic scheduling. Preserve correctness for append frontier, source/index atomicity, quiet deferral and crash recovery. The fix must bound live working set and prevent redundant catch-up while a prior pass is active; do not solve this by permanently disabling watching, broadening loss windows, or weakening authority proofs. Prove exact recovery/cursor behavior after daemon restart.","acceptance_criteria":"1. Reproduction measures memory high-water and bounded input work for a large, actively appended Codex JSONL. 2. One active file cannot schedule overlapping/redundant catch-up while its prior append pass is running. 3. Append ingestion retains no full historical payload/model beyond its operation boundary; RSS is bounded materially below service MemoryHigh on the reproduction. 4. Cursor/frontier/source/index correctness, restart recovery and failure rollback remain proven. 5. Focused tests and quick verification pass; live restart postflight does not reintroduce the hot loop.","notes":"Scoped 2026-07-13: reproduce and fix the active Codex JSONL append memory/catch-up incident in polylogue/sources/live plus focused tests only. I will use the existing #2841 cohort-memory harness, preserve cursor/frontier and rollback semantics, and avoid live archive or daemon mutation.","status":"closed","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-13T16:19:03Z","created_by":"Sinity","updated_at":"2026-07-13T16:39:42Z","started_at":"2026-07-13T16:19:59Z","closed_at":"2026-07-13T16:39:42Z","close_reason":"PR #2849 merged as 2b0221a98. Established byte-proven append cohorts now use durable replay metadata without historical full reads; incomplete/omitted-current chains classify then defer without cursor advance. Focused harness: 4 passed; devtools verify --quick: 15 checks passed. Live daemon remains stopped for operator postflight.","labels":["area:daemon","area:ingest","area:storage","delivery:G-live-performance","horizon:frontier"],"comments":[{"id":"019f6bee-2d1d-7b01-a481-a4089b02445e","issue_id":"polylogue-cnaj","author":"Sinity","text":"2026-07-16 closure-audit correction: keep closed. The packaged daemon is active (started 18:20:21 CEST) and the original append-reread hot loop did not recur in the observed live restart. The startup scan processed a 2.54 GB backlog; the relevant append chunk completed with append_files=2 and read_amp=0.614, and a later changed-file append completed with read_amp=0.0046. Current cgroup state at audit: memory.current about 1.53 GB, peak about 2.149 GB under a 2 GiB cap, zero oom/oom_kill, and zero current PSI. The backlog did hit the memory cap and current raw-frontier/CAS retries remain noisy, but those are separately owned by lkrc/yla8 and are not evidence that the cnaj historical-reread mechanism remains live. The earlier audit incorrectly treated the stale close-time sentence that the daemon remained stopped as current state.","created_at":"2026-07-16T17:16:39Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} {"_type":"issue","id":"polylogue-lkrc.2","title":"Repair remaining current unknown-origin ChatGPT heads","description":"Live postflight after the final legacy NULL-native-id copy-forward found nine current chatgpt-export sessions whose sessions.raw_id still points at a durable source.raw_sessions row typed origin=unknown-export with logical_source_key=unknown:\u003cnative-id\u003e. They are distinct from the three original lkrc raws: two siblings now point to canonical byte-proven copies and the legacy target points to 402915...; this residual cohort is a separate current-authority problem.","design":"Start from a fresh stopped-daemon census that joins current sessions to source.raw_sessions and production-normalizes each retained blob. Partition rows by existing revision/head/application/membership evidence; reuse an already-proven copy-forward route only when every source/index witness exactly matches its contract. Preserve original raw/blob/membership/head/application evidence, create a canonical replacement rather than relabelling historical raws, require a proof digest plus planned/applied receipt, and keep source-v7/v35 active-index compatibility. Do not treat retained non-current historical unknown heads as current mismatches.","acceptance_criteria":"1. Exact before census names every current session backed by unknown-export raw and distinguishes non-current retained history. 2. Every eligible row is repaired through a receipted, proof-bound, idempotent evidence-preserving path; ineligible shapes remain fail-closed with a durable reason. 3. Exact after census is zero current sessions whose raw origin/logical key disagrees with the production-normalized ChatGPT identity. 4. Focused real-route tests cover the observed evidence shapes, drift/rollback, generated active-index routing, and source-v7 compatibility; quick verification passes. 5. Live use follows a verified backup, stopped daemon, fresh dry proof, immutable receipt, and restart postflight.","notes":"Discovered 2026-07-13 after successful live legacy child repair receipt legacy-native-repair-20260713T160800Z.jsonl. Exact initial current cohort raw IDs: 3c144e4b6eccf6c65368488be8c952a510a50ed86deb9c93453b1a0dd08a55b2, 773bbbf1b92e763a0e85d1c798f127d94aa1e0f70b6e91978bcdd7cfbecc078d, 2af730ea7ca773cbb1983498d3103616e7309c41593cafa8e781a3eb151eca3b, bd47782eea0579a4bcba6d5b51670e4f71a80cf1473f38ee2536d07afb2ff1e0, 6567faf1da05d51ab8343fba6334602eef120f6b39ca1edb884a71edabe90d0d, 27527c1586e4e0105ec2a73c2206709af1ce74df0c0bb4dea24069f350644538, f43a203e159d29f403cca7123fb95c83ab3169f27978b7caa029c6496a0309e6, c10658915c27d74517c5d6f941247007564275d3d9360b2290683feb6593ee4b, 88aefc84afb181135c76a724b361ef21a9aa856f2a1ef117511e08fdceba2785.\nRead-only stopped-daemon census 2026-07-13: correct raw f43a203e159… to f43a203a359d29f403cca7123fb95c83ab3169f27978b7caa029c6496a0309e6. All nine old heads are unknown-export/native_id NULL/full+byte_proven/gen0 with one selected-baseline app and production parser identity match. Safe common rekey candidates: 773bbbf1…, bd47782e…, f43a203a… (no canonical head); 2af730ea…, 27527c15… (exact-equal semantic canonical witnesses). Fail-closed: 6567faf1… and c106589… semantic canonical hash conflicts (c106 diverges message 523); 3c144e… superseded_equivalent membership plus canonical hash conflict (diverges message 1333); 88aefc84… current reparse hash drift/incompatible canonical byte head. Existing actuators correctly reject all. Implement a new sibling byte-proven-browser-rekey actuator only for the five exact shapes; preserve all old/semantic evidence and record ineligible reasons for the four.\n2026-07-13: Claimed for isolated implementation of the sibling evidence-preserving byte-proven browser rekey actuator. Scope is exactly five proof-approved shapes; four observed conflict/drift shapes remain fail-closed. No live archive or daemon mutation is authorized by this implementation lane.\n2026-07-13: implementation merged in PR #2850 / master 64f4a00e8. The new repair_byte_proven_browser_capture_null_native_ids actuator is intentionally limited to the five proof-approved byte-proven NULL-native shapes. Verification: devtools verify --quick; focused byte-rekey matrix 10 passed. No live archive or daemon mutation occurred. Remaining scope is the parent-run stopped-daemon dry proof/apply/postflight, including durable reasons for the four ineligible rows.\n2026-07-14 status check (no live archive touched): re-verified the code portion of this bead is complete on current master (PR #2850 / 64f4a00e8, repair_byte_proven_browser_capture_null_native_ids). Confirmed via the existing 10-case focused byte-rekey matrix (test_byte_proven_browser_rekey_*) plus this session's own re-run: devtools test tests/unit/storage/test_browser_capture_origin_repair.py -k \"conflict or record_conflict\" -\u003e 11 passed. No further code change made or needed for this bead specifically in PR #2877 -- that PR's lkrc.3 work builds ON TOP of this bead's actuator (re-runs its exact eligibility proof) rather than modifying it. Remaining scope per this bead's own notes (\"parent-run stopped-daemon dry proof/apply/postflight, including durable reasons for the four ineligible rows\") is entirely live-execution, reserved for the operator; the \"durable reasons for the four ineligible rows\" portion is now directly actionable via record_browser_canonical_authority_conflict_blockers (PR #2877, polylogue-lkrc.3) once the operator runs it live.","status":"closed","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-13T16:13:11Z","created_by":"Sinity","updated_at":"2026-07-14T23:12:16Z","started_at":"2026-07-13T16:31:27Z","closed_at":"2026-07-14T23:12:16Z","labels":["area:browser","area:sources","area:storage","delivery:A-trust-floor","horizon:frontier"],"dependencies":[{"issue_id":"polylogue-lkrc.2","depends_on_id":"polylogue-lkrc","type":"supersedes","created_at":"2026-07-15T01:12:16Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-lkrc.1","title":"Copy forward legacy browser raw missing native identity","description":"After PR #2839 hardens browser-origin copy-forward proofs, the final live lkrc target 282983b4ec87c080fd60c31d9ebaa415a38f57c8f57bb22cdeda1b7906aca2c0 correctly refuses because its durable unknown-export raw has native_id=NULL, even though its retained browser-capture bytes parse to ChatGPT session 6a149c9e-2910-83eb-a93b-e6805f9f94f8. The row must not be relabelled or mutated in place.","design":"Add a separate, explicitly named evidence-preserving legacy-native-missing copy-forward route. It may accept native_id=NULL only as the exact legacy evidence shape, not as a general relaxation: prove raw origin=unknown-export, browser-capture provenance, native_id NULL, source/blob-ref path/hash/size agreement, complete singleton census, quarantined full envelope, production parse yields exactly one canonical ChatGPT session, canonical semantic authority and all applications/memberships/head witnesses match, and no competing old/canonical applications exist. Create a new canonical raw/application/receipt with parsed native identity; never update/delete the old raw/blob/head/application/membership. Planned/applied receipt records legacy-null witness and parser-derived native ID; locked reproof/CAS is all-or-nothing; reapply idempotent. Keep source-v7 compatibility.","acceptance_criteria":"1. Real-route fixture with legacy native_id NULL is ineligible to ordinary copy-forward but eligible only to the dedicated actuator after every listed witness is proven. 2. Any non-NULL wrong native, origin/path/blob/census/parser/session/head/application/timestamp/frontier/sibling drift fails before source write. 3. Apply makes a new correctly typed canonical raw and leaves all old evidence byte-for-byte unchanged; receipt proves the legacy-null witness and parsed identity. 4. Reapply is idempotent; planned/apply mismatch or post-proof failure rolls back. 5. Focused tests + quick pass; live use only after fresh full backup, stopped daemon, read-only dry run, exact receipt, apply, and postflight zero mismatched heads.","notes":"2026-07-13: Claimed after PR #2839 merged as db586289e. Ordinary actuator is deliberately fail-closed for native_id=NULL; this child owns the separate legacy-only copy-forward path. Implementation must preserve source-v7 compatibility and not mutate the old raw.","status":"closed","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-13T12:54:50Z","created_by":"Sinity","updated_at":"2026-07-13T16:16:35Z","started_at":"2026-07-13T13:03:10Z","closed_at":"2026-07-13T16:16:35Z","close_reason":"Live repair applied with receipt legacy-native-repair-20260713T160800Z.jsonl; source-v7-compatible v35 artifact verified; rerun reports already_repaired.","labels":["area:browser","area:sources","area:storage","delivery:A-trust-floor","horizon:frontier"],"dependencies":[{"issue_id":"polylogue-lkrc.1","depends_on_id":"polylogue-lkrc","type":"parent-child","created_at":"2026-07-13T14:54:49Z","created_by":"Sinity","metadata":"{}"}],"comments":[{"id":"019f5bb0-a6fb-7469-a902-893404f4e28f","issue_id":"polylogue-lkrc.1","author":"Sinity","text":"2026-07-13 implementation update: legacy-only NULL-native route now refuses a pre-existing canonical head, requires exactly one old raw membership key and payload blob reference, and stages source copy-forward plus index authority transition in one attached-source transaction. A regression injects a failure after source staging and proves old source/index rows remain unchanged with a planned-only receipt. Verification: devtools test tests/unit/storage/test_browser_capture_origin_repair.py -k legacy_browser_native_id (13 passed); devtools test tests/unit/storage/test_browser_capture_origin_repair.py tests/unit/cli/test_archive_maintenance_cli.py -k 'legacy_browser_native_id or rejects_legacy_raw_without_native_id' (15 passed); devtools verify --quick (passed). Pending independent re-audit; no live archive actuator has been run.","created_at":"2026-07-13T13:35:32Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} -{"_type":"issue","id":"polylogue-lkrc","title":"Converge raw evidence authority through one proof-driven reconciler","description":"Polylogue has accumulated separate repair actuators and incident Beads for origin-mismatched browser raws, competing canonical heads, duplicate raw identities, replaced snapshots requiring reacquisition, quarantined accepted raws, and superseded revisions. These are not independent product capabilities. They are states of one raw-evidence authority lifecycle whose invariant is that every accepted materialized head is backed by a typed, byte-identified, provenance-authorized raw revision—or is held in an explicit unresolved/conflict/reacquisition state.","design":"Create one RawAuthorityReconciler over the existing raw frontier projection, raw revision authority types, OriginSpec evidence, and repair proof/receipt machinery. It enumerates every accepted/materialized head and classifies it into proven-current, safely rekeyable/equivalent, duplicate-alias, superseded, missing-bytes/reacquire, conflicting-authority/needs-judgment, unresolved-provenance, or corrupt. A canonical plan schema carries witnesses, source/head hashes, expected identities, authority, intended actuator, and preconditions. Apply uses one plan-authorize-apply-receipt-postflight protocol with locked atomic receipts and compare-and-swap revalidation; existing browser-origin, duplicate-identity, quarantined-head, and superseded-snapshot functions become actuator strategies behind it or are deleted. Safe deterministic repairs may converge automatically through the daemon after quiet/proof gates; conflicting content never auto-wins and instead emits a durable judgment request/blocker. Reacquisition is a durable obligation linked to the retained receiver/source artifact and must prove byte identity before promotion. The reconciler reports complete counts and stable refs across all states and is idempotent/restartable. yla8 remains the distinct prevention invariant for replay ordering; this bead repairs and continuously audits the frontier rather than duplicating that write-path rule.","acceptance_criteria":"1. One census/plan covers origin mismatch, duplicate identity, quarantined accepted raw, superseded snapshot, missing/replaced bytes, and competing canonical authority with mutually exclusive typed states and stable evidence refs. 2. One plan-authorize-apply-receipt-postflight contract drives every actuator; grep finds no independent proof-digest/receipt lifecycle for browser-origin versus duplicate-identity repairs. 3. Deterministically equivalent/rekeyable/duplicate cases converge idempotently and restartably; compare-and-swap revalidation prevents stale-plan writes. 4. Conflicting byte/content authority cannot auto-select a winner and produces a durable queryable judgment blocker; an operator assertion can resume the same plan. 5. Missing bytes create a durable reacquisition obligation and promote only after origin/identity/hash proof; replaced receiver artifacts are not silently lost. 6. The known lkrc/lkrc.3, 57rp, t0dy, and quarantined/superseded fixtures all pass through the single reconciler, and a stopped-daemon live postflight leaves zero unreported frontier gaps. 7. Readiness/status expose state counts and remediation refs; known-sidecar or accepted-index status alone cannot report healthy. 8. OriginSpec supplies authority rules and yla8 replay-order protections remain intact; mutation tests fail if either is bypassed.","notes":"2026-07-13 live v35 postflight: verified full_evidence backup receipt at /realm/staging/polylogue-sqlite/recovery/lkrc-v35-20260713T042736Z/polylogue-archive-20260713T042738Z/verification-receipt.json (all five SQLite tiers, 26,600 blobs). Exact v7/index-v35/user-v6 artifact completed all watcher catch-up chunks with no recurrence of membership replay cannot retire an unrelated accepted head. Stopped-daemon census found 11 unknown-export-\u003eChatGPT session/raw mismatches. The three lkrc raws are quarantined full singleton censuses with canonical membership decision NULL and exact old unknown-key selected-baseline receipts; actuator now requires that narrow dual witness. The other 8 are excluded: 7 byte_proven unknown raws without membership/census, 1 byte_proven superseded-equivalent membership; separate follow-up required.\n2026-07-13 adversarial loop iteration 5 reached its cap with unresolved P0 proof gaps; do not merge/apply #2839 head 3b0ca3f08. Real residuals: (1) semantic canonical and historical sibling source envelopes omit capture_mode; require canonical provider when schema has field, with v7 fallback. (2) original unknown raw blob_ref.source_path is not bound to raw source_path in preflight/locked reproof. (3) original unknown raw native_id is not bound to reparsed provider session id preflight/locked reproof. (4) restore_canonical_head exact-byte route omits native_id, source_index, capture_mode, predecessor/append envelope fields; normalize conditional full-envelope proof for exact/semantic/sibling paths. Lower severity: historical supersession decided_at_ms accepts negative values. Iteration-5 reviewer found these against the current 31-test terminal closure; no live mutation after findings. Further implementation plus an operator-authorized review cycle is required before merge/apply.\n2026-07-14 code-verification pass (no live archive touched): re-checked the \"adversarial loop iteration 5\" proof gaps recorded in this bead's prior note against current master (031d8d183) source. All 3 named residual gaps -- (1) capture_mode binding, (2) blob_ref.source_path binding, (3) native_id binding into the preflight/locked reproof witness -- are already present in _browser_origin_source_envelope_is_exact (polylogue/storage/repair.py), which every browser-origin repair path (exact-canonical, semantic, and the restore_canonical_head route) now shares. Confirmed these landed via PRs #2843/#2847/#2848/#2850 (all merged after the iteration-5 note was written) by git log/git show on the relevant commits. AC1 (new browser captures acquire chatgpt-export origin, not unknown-export) is already covered by test_streaming_sized_browser_capture_json_uses_native_payload_detection in tests/unit/sources/test_live_batch_support.py, which asserts `SELECT origin FROM raw_sessions` == chatgpt-export for a fresh ingest.\nPR #2877 (branch feature/fix/raw-identity-repair-cluster) adds the evidence-packet + durable-blocker capability for this bead's dependent polylogue-lkrc.3 (the 4 sessions the exact-byte rekey actuator correctly refuses) -- see that bead's notes. AC4 (dynamic live census reports zero mismatches, or every unresolved conflict is an explicit durable blocking state) remains open pending a live-archive run of record_browser_canonical_authority_conflict_blockers, which this session does not perform (live-execution reserved for the operator). No code gap was identified beyond what #2877 adds; this bead's remaining scope is live-execution, not implementation.\n[2026-07-15 invariant-collapse pass] Expanded from the browser-origin incident into the shared raw-authority state machine evidenced by multiple separate repair classes in storage/repair.py. Supersedes lkrc.3, 57rp, and t0dy; their named live cases are regression/postflight inputs, not separate scheduled projects. Does not absorb yla8 because preventing stale replay is a different write-path invariant.\nLive evidence 2026-07-15 from MCP readiness_check: raw_frontier_integrity reported 1,890 broken active heads among 18,347 checked, 40 ingest cursors committed past accepted raw material, and 34 cursor/head authority rows not comparable. This is current measured debt, not a repair instruction; preserve the snapshot/frame and classify through the proof-driven reconciler before any cursor reset or replay mutation.\n2026-07-15 yla8 read-only preflight sharpened the live failure: packaged build 20d703e (source11/index36/user8) reports 1,890 broken active seeds, 40 cursor-ahead rows, 34 incomparable authority rows, and 15,264 direct / 21,398 expanded replay candidates. Journal shows ordinary convergence replaying exactly 2 logical sources per pass while the candidate count rose from 11,717 to 15,264 over four hours. Treat current rows as an immutable-frame census for RawAuthorityReconciler classification; do not reset cursors or run broad replay. hjpx owns the accepted-plan-to-fixed-point execution defect and is now P0 discovered from the failed yla8 gate.\n2026-07-16 implementation pass: owning the coherent lkrc/hjpx.1/lkrc.4 raw-authority cluster from fresh origin/master. Scope is the single reconciler/immutable-plan conservation and the production multi-session divergence regression now observed in packaged ordinary catch-up. Preserve yla8 fail-closed replay protections; no live cursor reset, force replay, evidence deletion, manual SQL repair, or live apply before reviewed code, verified backup, quiescent census, and explicit authorization. First deliverable is a production-route failing fixture and read-only live evidence.","status":"in_progress","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-12T23:50:53Z","created_by":"Sinity","updated_at":"2026-07-16T19:20:39Z","started_at":"2026-07-16T19:20:39Z","metadata":{"frontier":"active","frontier_program_ref":"polylogue-1xc"},"labels":["area:browser","area:sources","area:storage","delivery:A-trust-floor","horizon:frontier"],"dependencies":[{"issue_id":"polylogue-lkrc","depends_on_id":"polylogue-1xc","type":"parent-child","created_at":"2026-07-15T01:15:39Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-lkrc","depends_on_id":"polylogue-1xc.13","type":"relates-to","created_at":"2026-07-15T06:25:34Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-lkrc","depends_on_id":"polylogue-2qx","type":"related","created_at":"2026-07-15T01:09:46Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-lkrc","depends_on_id":"polylogue-b5l.1","type":"relates-to","created_at":"2026-07-15T20:42:23Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-lkrc","depends_on_id":"polylogue-yla8","type":"blocks","created_at":"2026-07-15T01:09:45Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-lkrc","depends_on_id":"polylogue-yla8.10","type":"discovered-from","created_at":"2026-07-13T01:50:54Z","created_by":"Sinity","metadata":"{}"}],"comments":[{"id":"019f6407-6e37-7464-b88f-e043f6e0c88b","issue_id":"polylogue-lkrc","author":"Sinity","text":"[Dogfood 2026-07-15 / F-004] A named growing Codex source had an excluded cursor after five failures, later acquired raws unparsed, and a stale indexed session. Archive census showed 3,821 excluded cursors, 1,890 broken heads, 41 cursor-ahead rows, and 34 authority gaps. polylogue-1xc.13 owns the named-source acquisition-to-searchable projection and excluded-not-idle semantics. This reconciler remains the owner of underlying authority classification and repair population, so the beads are related rather than duplicating actuators.","created_at":"2026-07-15T04:27:16Z"}],"dependency_count":1,"dependent_count":0,"comment_count":1} +{"_type":"issue","id":"polylogue-lkrc","title":"Converge raw evidence authority through one proof-driven reconciler","description":"Polylogue has accumulated separate repair actuators and incident Beads for origin-mismatched browser raws, competing canonical heads, duplicate raw identities, replaced snapshots requiring reacquisition, quarantined accepted raws, and superseded revisions. These are not independent product capabilities. They are states of one raw-evidence authority lifecycle whose invariant is that every accepted materialized head is backed by a typed, byte-identified, provenance-authorized raw revision—or is held in an explicit unresolved/conflict/reacquisition state.","design":"Create one RawAuthorityReconciler over the existing raw frontier projection, raw revision authority types, OriginSpec evidence, and repair proof/receipt machinery. It enumerates every accepted/materialized head and classifies it into proven-current, safely rekeyable/equivalent, duplicate-alias, superseded, missing-bytes/reacquire, conflicting-authority/needs-judgment, unresolved-provenance, or corrupt. A canonical plan schema carries witnesses, source/head hashes, expected identities, authority, intended actuator, and preconditions. Apply uses one plan-authorize-apply-receipt-postflight protocol with locked atomic receipts and compare-and-swap revalidation; existing browser-origin, duplicate-identity, quarantined-head, and superseded-snapshot functions become actuator strategies behind it or are deleted. Safe deterministic repairs may converge automatically through the daemon after quiet/proof gates; conflicting content never auto-wins and instead emits a durable judgment request/blocker. Reacquisition is a durable obligation linked to the retained receiver/source artifact and must prove byte identity before promotion. The reconciler reports complete counts and stable refs across all states and is idempotent/restartable. yla8 remains the distinct prevention invariant for replay ordering; this bead repairs and continuously audits the frontier rather than duplicating that write-path rule.","acceptance_criteria":"1. One census/plan covers origin mismatch, duplicate identity, quarantined accepted raw, superseded snapshot, missing/replaced bytes, and competing canonical authority with mutually exclusive typed states and stable evidence refs. 2. One plan-authorize-apply-receipt-postflight contract drives every actuator; grep finds no independent proof-digest/receipt lifecycle for browser-origin versus duplicate-identity repairs. 3. Deterministically equivalent/rekeyable/duplicate cases converge idempotently and restartably; compare-and-swap revalidation prevents stale-plan writes. 4. Conflicting byte/content authority cannot auto-select a winner and produces a durable queryable judgment blocker; an operator assertion can resume the same plan. 5. Missing bytes create a durable reacquisition obligation and promote only after origin/identity/hash proof; replaced receiver artifacts are not silently lost. 6. The known lkrc/lkrc.3, 57rp, t0dy, and quarantined/superseded fixtures all pass through the single reconciler, and a stopped-daemon live postflight leaves zero unreported frontier gaps. 7. Readiness/status expose state counts and remediation refs; known-sidecar or accepted-index status alone cannot report healthy. 8. OriginSpec supplies authority rules and yla8 replay-order protections remain intact; mutation tests fail if either is bypassed.","notes":"2026-07-13 live v35 postflight: verified full_evidence backup receipt at /realm/staging/polylogue-sqlite/recovery/lkrc-v35-20260713T042736Z/polylogue-archive-20260713T042738Z/verification-receipt.json (all five SQLite tiers, 26,600 blobs). Exact v7/index-v35/user-v6 artifact completed all watcher catch-up chunks with no recurrence of membership replay cannot retire an unrelated accepted head. Stopped-daemon census found 11 unknown-export-\u003eChatGPT session/raw mismatches. The three lkrc raws are quarantined full singleton censuses with canonical membership decision NULL and exact old unknown-key selected-baseline receipts; actuator now requires that narrow dual witness. The other 8 are excluded: 7 byte_proven unknown raws without membership/census, 1 byte_proven superseded-equivalent membership; separate follow-up required.\n2026-07-13 adversarial loop iteration 5 reached its cap with unresolved P0 proof gaps; do not merge/apply #2839 head 3b0ca3f08. Real residuals: (1) semantic canonical and historical sibling source envelopes omit capture_mode; require canonical provider when schema has field, with v7 fallback. (2) original unknown raw blob_ref.source_path is not bound to raw source_path in preflight/locked reproof. (3) original unknown raw native_id is not bound to reparsed provider session id preflight/locked reproof. (4) restore_canonical_head exact-byte route omits native_id, source_index, capture_mode, predecessor/append envelope fields; normalize conditional full-envelope proof for exact/semantic/sibling paths. Lower severity: historical supersession decided_at_ms accepts negative values. Iteration-5 reviewer found these against the current 31-test terminal closure; no live mutation after findings. Further implementation plus an operator-authorized review cycle is required before merge/apply.\n2026-07-14 code-verification pass (no live archive touched): re-checked the \"adversarial loop iteration 5\" proof gaps recorded in this bead's prior note against current master (031d8d183) source. All 3 named residual gaps -- (1) capture_mode binding, (2) blob_ref.source_path binding, (3) native_id binding into the preflight/locked reproof witness -- are already present in _browser_origin_source_envelope_is_exact (polylogue/storage/repair.py), which every browser-origin repair path (exact-canonical, semantic, and the restore_canonical_head route) now shares. Confirmed these landed via PRs #2843/#2847/#2848/#2850 (all merged after the iteration-5 note was written) by git log/git show on the relevant commits. AC1 (new browser captures acquire chatgpt-export origin, not unknown-export) is already covered by test_streaming_sized_browser_capture_json_uses_native_payload_detection in tests/unit/sources/test_live_batch_support.py, which asserts `SELECT origin FROM raw_sessions` == chatgpt-export for a fresh ingest.\nPR #2877 (branch feature/fix/raw-identity-repair-cluster) adds the evidence-packet + durable-blocker capability for this bead's dependent polylogue-lkrc.3 (the 4 sessions the exact-byte rekey actuator correctly refuses) -- see that bead's notes. AC4 (dynamic live census reports zero mismatches, or every unresolved conflict is an explicit durable blocking state) remains open pending a live-archive run of record_browser_canonical_authority_conflict_blockers, which this session does not perform (live-execution reserved for the operator). No code gap was identified beyond what #2877 adds; this bead's remaining scope is live-execution, not implementation.\n[2026-07-15 invariant-collapse pass] Expanded from the browser-origin incident into the shared raw-authority state machine evidenced by multiple separate repair classes in storage/repair.py. Supersedes lkrc.3, 57rp, and t0dy; their named live cases are regression/postflight inputs, not separate scheduled projects. Does not absorb yla8 because preventing stale replay is a different write-path invariant.\nLive evidence 2026-07-15 from MCP readiness_check: raw_frontier_integrity reported 1,890 broken active heads among 18,347 checked, 40 ingest cursors committed past accepted raw material, and 34 cursor/head authority rows not comparable. This is current measured debt, not a repair instruction; preserve the snapshot/frame and classify through the proof-driven reconciler before any cursor reset or replay mutation.\n2026-07-15 yla8 read-only preflight sharpened the live failure: packaged build 20d703e (source11/index36/user8) reports 1,890 broken active seeds, 40 cursor-ahead rows, 34 incomparable authority rows, and 15,264 direct / 21,398 expanded replay candidates. Journal shows ordinary convergence replaying exactly 2 logical sources per pass while the candidate count rose from 11,717 to 15,264 over four hours. Treat current rows as an immutable-frame census for RawAuthorityReconciler classification; do not reset cursors or run broad replay. hjpx owns the accepted-plan-to-fixed-point execution defect and is now P0 discovered from the failed yla8 gate.\n2026-07-16 implementation pass: owning the coherent lkrc/hjpx.1/lkrc.4 raw-authority cluster from fresh origin/master. Scope is the single reconciler/immutable-plan conservation and the production multi-session divergence regression now observed in packaged ordinary catch-up. Preserve yla8 fail-closed replay protections; no live cursor reset, force replay, evidence deletion, manual SQL repair, or live apply before reviewed code, verified backup, quiescent census, and explicit authorization. First deliverable is a production-route failing fixture and read-only live evidence.\n2026-07-17 PR #2962 closure implementation at edd68d240: the shared frontier now owns typed conflict disposition end to end. A conflicting browser head remains non-executable until its exact candidate judgment assertion is accepted and the blocker is resolved with disposition=retain_canonical_authority; the resulting immutable successor plan CAS-revalidates the complete competing-head witness, retains canonical authority, records supersession, retires the obsolete unknown-key head, and proves a terminal postcondition. Browser copy-forward/restore now also remove the obsolete head instead of leaving a corrupt residual frontier. Old incident receipt/mutator/CLI lifecycles were removed. Focused production-route verification: 183 passed across raw ledger, browser, quarantine, duplicate identity, daemon CLI, and maintenance CLI. Quick gate 20260717T000755Z-quick-1199839-9d926a5d: 16/16 green. Remaining closure boundary is the separately authorized stopped-daemon live gate in yla8; no live archive mutation was performed by this PR.","status":"in_progress","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-12T23:50:53Z","created_by":"Sinity","updated_at":"2026-07-17T00:09:42Z","started_at":"2026-07-16T19:20:39Z","metadata":{"frontier":"active","frontier_program_ref":"polylogue-1xc"},"labels":["area:browser","area:sources","area:storage","delivery:A-trust-floor","horizon:frontier"],"dependencies":[{"issue_id":"polylogue-lkrc","depends_on_id":"polylogue-1xc","type":"parent-child","created_at":"2026-07-15T01:15:39Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-lkrc","depends_on_id":"polylogue-1xc.13","type":"relates-to","created_at":"2026-07-15T06:25:34Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-lkrc","depends_on_id":"polylogue-2qx","type":"related","created_at":"2026-07-15T01:09:46Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-lkrc","depends_on_id":"polylogue-b5l.1","type":"relates-to","created_at":"2026-07-15T20:42:23Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-lkrc","depends_on_id":"polylogue-yla8","type":"blocks","created_at":"2026-07-15T01:09:45Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-lkrc","depends_on_id":"polylogue-yla8.10","type":"discovered-from","created_at":"2026-07-13T01:50:54Z","created_by":"Sinity","metadata":"{}"}],"comments":[{"id":"019f6407-6e37-7464-b88f-e043f6e0c88b","issue_id":"polylogue-lkrc","author":"Sinity","text":"[Dogfood 2026-07-15 / F-004] A named growing Codex source had an excluded cursor after five failures, later acquired raws unparsed, and a stale indexed session. Archive census showed 3,821 excluded cursors, 1,890 broken heads, 41 cursor-ahead rows, and 34 authority gaps. polylogue-1xc.13 owns the named-source acquisition-to-searchable projection and excluded-not-idle semantics. This reconciler remains the owner of underlying authority classification and repair population, so the beads are related rather than duplicating actuators.","created_at":"2026-07-15T04:27:16Z"}],"dependency_count":1,"dependent_count":0,"comment_count":1} {"_type":"issue","id":"polylogue-5ucz","title":"Fast-forward the live v32 index to v35 without raw replay","description":"The canonical 32 GiB index is healthy at user_version=32 but current code requires v35. A raw reparse is unnecessary and expensive: v33 widens one CHECK, v34 adds one index plus the current delegations view rewrite, and v35 changes three FTS tokenizers/write folds. Build and prove a clone-first fast-forward that leaves the original untouched, rebuilds only derived FTS tables from normalized source tables, and supports atomic blue-green activation with rollback.","design":"Implement an evidence-harness and operator actuator on a fresh branch from origin/master. Quiesce the user daemon; checkpoint/copy the v32 index using WAL-consistent handling and a Btrfs reflink under a contained single-operation scope. Apply exact canonical v33/v34/v35 DDL deltas to the clone, including the current delegations view definition, rebuilding all three contentless FTS tables with the canonical v35 tokenizers and folded write path through existing repair machinery. Set user_version=35 only after every mutation succeeds. Validate quick_check, foreign keys, exact canonical DDL, stable structural row counts, FTS population counts/folded-query smoke, and readiness on the clone. Emit phase/timing/hash/count/resource receipts. Activation is a same-filesystem atomic blue-green swap with retained rollback target; restart and postflight only after clone proof. No raw parse or durable-tier mutation.","acceptance_criteria":"1. A small v32 fixture proves exact 32→35 deltas, current delegations view, all three canonical FTS definitions/content, user_version-last behavior, and rollback on injected failure without raw parsing. 2. The live daemon is quiesced and the 32 GiB original remains byte/path preserved while a WAL-consistent reflink clone is created; receipts record source identity, sidecars/checkpoint state, timings, sizes, and resource envelope. 3. Clone mutation applies v33/v34/v35 canonical deltas and rebuilds messages_fts/work_events_fts/threads_fts from their source tables using current v35 folding/tokenizers; no session/message/block/source raw replay occurs. 4. Clone gates pass: integrity_check or quick_check as designed, foreign_key_check=0, canonical DDL exactness, unchanged sessions/messages/blocks and other structural counts, expected FTS counts, folded-query smoke, user_version=35, and current runtime readiness. 5. Only after a clone-only report is reviewed green, activation atomically swaps the canonical index to the proven clone on the same filesystem, retains the v32 rollback target, restarts the daemon, and proves bounded journal/readiness/query smoke. Any failure before activation leaves v32 canonical; any post-activation failure rolls back atomically. 6. Exact commands, timings, hashes/counts, PSI/RSS/IO samples, receipt paths, and no-raw-reparse evidence are attached. No v35 rebuild through ordinary raw ingestion.","notes":"Deployment/postflight completion:\n- Sinnix polylogue input advanced eff7c2a→58691ab and canonical devshell switch completed; deployed package /nix/store/acgsm0akngfg6jg23cllnx22xxl83hgy-python3.13-polylogue-0.1.0.\n- Current runtime also required durable user.db v4→v6. Used verified user_overlays backups at /realm/staging/polylogue-sqlite/recovery/user-v6-20260713/polylogue-archive-20260712T230342Z and /realm/staging/polylogue-sqlite/recovery/user-v6-step2-20260713/polylogue-archive-20260712T230517Z. Runner correctly refused stale-manifest reuse between migration steps.\n- Final: index user_version=35; user user_version=6; user quick_check=ok; foreign_key_check empty; annotation_schemas, annotation_batches, context_deliveries present; delegation.discourse v1 registered.\n- polylogued active/running PID 1943471, NRestarts=0; no storage schema mismatch; 8/8 live sources; browser spool ready; ports 8765/8766 owned by the integrated daemon. Receipt postflight field updated and hash refreshed.\n- PR #2804 merged as 07fbbeeca1c298aae6a964712374d4c40aa81e1f. GitHub-hosted checks did not start because the account is billing-locked; local owning tests and two quick gates were green, and no review threads/actionable bot findings existed.","status":"closed","priority":0,"issue_type":"task","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-12T19:57:48Z","created_by":"Sinity","updated_at":"2026-07-12T23:06:53Z","started_at":"2026-07-12T19:57:54Z","closed_at":"2026-07-12T23:06:53Z","close_reason":"Delivered and live: clone-first no-raw v32→v35 activation proven, deployed v35 runtime plus verified user v6 migrations, stable daemon/query postflight, retained v32 rollback, PR #2804 merged.","labels":["area:ops","area:storage","area:test","delivery:B-storage-rebuild-bytes","horizon:frontier","lane:storage-rebuild-scale","spine"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-jlme.2","title":"Fail closed and preserve first-party auth for browser backfills","description":"Live deployment of PR #2771 proved a provider-contract failure: an authenticated ChatGPT UI with visible history returned HTTP 200 total=0/items=[] to the extension background adapter, which accepted the empty inventory as complete. The frontend itself requests the same inventory family with first-party page context and visibly receives history. A background fetch must not silently convert missing page/auth/account context into a successful empty archive delta. Fix ChatGPT and audit Claude transport while honoring provider controls, keeping secrets ephemeral, and avoiding foreground activation or broad live crawling.","design":"Evidence first: capture a bounded frontend inventory request through CDP and compare only header names, initiator/context, status, and response shape with the extension request; redact all credential values. Rank cookie context, account header, device/session token, and execution-world differences before choosing a transport. Implement a main-world/page bridge or equivalent ephemeral authenticated transport so provider-native inventory/fetch calls execute in the first-party context. The service worker remains coordinator/storage owner. Bridge messages use request IDs, a strict allowlist of provider-relative endpoints/methods, fixed timeouts, response-size bounds, and fail-closed shape/auth/challenge handling. Never persist or log tokens/cookies/account identifiers. Provider 200/empty must be distinguished from trustworthy empty inventory using authenticated-context proof or consistency checks. Audit Claude under the same contract and share the transport abstraction where viable. No foreground activation.","acceptance_criteria":"1. A production-path fixture reproduces HTTP 200 empty inventory from an unauthenticated/background context while a page-context fixture has history; the adapter refuses to mark the former complete. 2. ChatGPT inventory and native fetch can use a strictly allowlisted first-party page/main-world bridge without persisting or logging credentials, and auth/challenge/timeout/oversize/drift fail closed. 3. Claude transport is either moved to the same authenticated-context mechanism or has evidence-backed proof its existing background requests carry sufficient context; no silent empty success. 4. Memory/fake-IndexedDB coordinator tests prove a rejected empty inventory remains paused/actionable and resumes without duplicate capture. 5. Packaged service-worker proof exercises bridge request/response correlation and confirms no foreground tab activation. 6. Bounded live deployment against the owned private-visible profile returns a nonzero inventory count consistent with visible history, then a conservative job starts under configured rate limits. No archive rebuild or v35 work.","notes":"Discovered after merge 07ea5f2d0 / PR #2771. Initial live evidence: ChatGPT background request /backend-api/conversations?offset=0\u0026limit=100\u0026order=updated returned 200 total=0/items=[]; frontend resource used offset=0\u0026limit=28\u0026order=updated\u0026is_archived=false\u0026is_starred=false while sidebar visibly showed history. Investigation may inspect credential header names but must never record values.\nClosure evidence 2026-07-12:\n- PR #2773 squash-merged as 901825ec4acbf278ad184a004acf604048508174.\n- Production transport executes strict structured operations directly in the authenticated first-party MAIN world; no postMessage trust or credential persistence. ChatGPT traverses all archived/starred partitions; Claude pins the exact UI-selected organization. Responses are streamed under a 32 MiB cap and temporary background tabs are lifecycle-bounded without foreground activation.\n- Verification: browser-extension npm test 158/158; focused 58/58; npm run lint clean; npm run validate manifest v0.1.0 valid; devtools verify --quick 15/15 (20260712T202234Z-quick-3863858-c3dff574). Adversarial and Codex findings were fixed; all substantive threads resolved. GitHub-hosted jobs failed before runner allocation (empty runner/steps), while GitGuardian and CodeRabbit passed.\n- Bounded live deployment in private-visible profile with cutoff 2026-04-23: ChatGPT inventory_complete=true with 477 eligible candidates and durable ACK polylogue-ext-mri9iyo5-9v0liypp; Claude inventory_complete=true over 900 provider records with 26 post-cutoff candidates, exact selected organization pinned, and durable ACK polylogue-ext-mri9j03e-ol7rdst0. Both live jobs run at base cadence 10s, max 800 provider cost units/day, concurrency/captures-per-wake 1, retaining Retry-After, full jitter, and circuit breaker behavior. No auth or rate-limit failure.\n- Live receiver compatibility probe posted the exact stored 1,174,387-byte envelope and received HTTP 202 with a 64-character content_hash matching the extension SHA-256.\n- The original false-zero job was cancelled and never resumed. Its in-profile ledger was subsequently lost when earlyoom killed Chrome and the private-start helper destructively re-seeded the profile; this is recorded honestly rather than reconstructed. Follow-ups: polylogue-jlme.3 (stale receiver contract handling) and polylogue-jlme.4 (ledger-preserving browser recovery/profile reseed).\n- Host evidence: earlyoom acted at ~2-3% available RAM with swap exhausted and killed Chrome renderers plus many 1-2.4 GiB codebase-memory-mcp processes. The backfill itself remained single-request and was not the pressure source.\nPost-closure live continuation: Claude job backfill-claude-ai-1783888873491-d7l8y reached COMPLETE with 25 durable captures, one explicit no_turns, zero retry/error/operator-action backlog, and final ACK polylogue-ext-mri9m6p9-0x0xjckx. ChatGPT job backfill-chatgpt-1783888873491-bdvr57 remained RUNNING at 17/477 durable captures, zero retry/error/operator-action backlog, under the requested 10s/800-cost/one-capture policy. The diagnostic popup and extension-created Claude tab were closed; the pre-existing active ChatGPT tab remained foreground and was never programmatically activated. The merged feature worktree is intentionally retained temporarily because Chrome loaded the unpacked extension from that exact path; removing it while the background job runs would break MV3 worker restart.","status":"closed","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-12T19:32:21Z","created_by":"Sinity","updated_at":"2026-07-12T20:48:40Z","started_at":"2026-07-12T19:32:27Z","closed_at":"2026-07-12T20:47:22Z","close_reason":"Delivered by PR #2773 / merge 901825ec with every acceptance criterion verified locally and bounded live ChatGPT+Claude inventories plus durable receiver ACKs.","labels":["area:ingest","area:web","delivery:G-live-performance","horizon:frontier","lane:capture-reliability","spine"],"dependencies":[{"issue_id":"polylogue-jlme.2","depends_on_id":"polylogue-jlme","type":"parent-child","created_at":"2026-07-12T21:32:21Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-jlme.2","depends_on_id":"polylogue-jlme.1","type":"discovered-from","created_at":"2026-07-12T21:32:22Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-yla8.10","title":"Repair accepted heads backed by untyped single-session raws","description":"The exact live v32 frontier has three active ChatGPT byte heads whose accepted_raw_id exists durably but still has no typed revision envelope: logical_source_key/source_revision are NULL, revision_kind=unknown, revision_authority=quarantined. The accepted index head/session is therefore not reconstructibly bound to source authority, and raw-frontier integrity correctly fails. Cursor-only yla8.6 repair cannot affect these rows. The retained v32 package at commit 3423d3c would classify a repeated single-session full as QUARANTINED, so ordinary re-acquisition alone remains false-green. Add a typed, evidence-preserving path that repairs this exact state without deleting or laundering raw/blob/head/receipt/session evidence.","design":"Recognize only the narrow already-accepted-untyped state: one current raw_revision_head and session raw_id agree on the same retained raw; source raw is unknown/quarantined with no prior logical/source binding; retained blob bytes normalize through the production ingest fallback-timestamp path to exactly the head session identity/content hash; SHA-256(payload) equals accepted_source_revision; byte length equals accepted frontier; raw row, raw_payload blob_ref, optional raw_artifact, origin, path, size, and source_index agree; the one immutable selected_baseline application receipt exactly equals the head including decided_at; and no competing head/application/membership/typed logical-key authority exists. Dry-run emits per-target and aggregate proof digests. Apply requires the exact digest/list and an explicit operator receipt path. Exclusively create and fsync a planned recovery receipt containing every witness, acquire ActiveWriterLease, open source.db as the sole writable main with index.db attached read-only, BEGIN IMMEDIATE once, reprove all targets, CAS-refine every envelope, reprove the terminal state, and commit all-or-nothing. Then fsync an applied record to the append-only operator receipt. Restart from a matching planned receipt is idempotent: exact already-bound rows finalize; any mismatch refuses. The existing immutable raw_revision_application proves prior acceptance and is cited, never mutated or duplicated. Do not weaken CAS, infer authority from raw_id alone, overwrite a typed envelope, misuse hook/ops tables, or delete evidence. Keep the actuator schema-v32-compatible and produce an exact v32-based build/artifact before live use.","acceptance_criteria":"1. Real-route fixture creates the exact invalid state through production write/receipt paths. Dry-run names each eligible raw, every witness, a per-target proof digest, and a deterministic aggregate digest without mutation; duplicate ids are rejected. 2. Apply requires that exact digest/list and an exclusive operator receipt path. It fsyncs planned evidence, acquires the writer lease, reproves under one source-main/index-readonly BEGIN IMMEDIATE transaction, CAS-refines all envelopes, reproves, commits all-or-nothing, and fsyncs applied terminal evidence. Raw/blob/session/head/content/message/FTS/application state is unchanged except the intended source authority columns. 3. Mutations for head/raw disagreement, missing or changed blob, blob-ref/artifact mismatch, byte-length/frontier drift, production-normalized parser/content-hash drift, wrong origin/session identity, competing head/application/typed revision/membership (including failed or ambiguous census), receipt/head field or decided_at drift, multi-session ambiguity, and pre-existing non-null envelope all fail closed with logical state unchanged. 4. Reapply with the matching applied receipt is idempotent. A planned-only receipt plus a partially/fully already-bound exact set resumes and finalizes; target/digest mismatch refuses. Injected proof/CAS/post-proof failures roll back the entire source batch and never leave a source binding without the pre-existing immutable application proof plus planned operator receipt. 5. Focused real-route storage and CLI tests pass, including anti-vacuity mutations. No schema changes; build the actuator from an exact INDEX_SCHEMA_VERSION=32 base containing all authority fixes through #2723, record build commit/hash, and run devtools verify --quick. 6. Live postflight only after merge and exact v32 artifact: stop daemon, verify source/user backup and dynamic census, dry-run exactly the current invalid raws, apply with stored operator receipt, then cursor-only yla8.6 repair/catch-up. Final exact census is 0 invalid heads and 0 cursor-ahead; explain incomparable gaps; source/index/hash/count parity and bounded journal are clean; controlled sanitized-copy append advances exactly once without shrink. No rebuild is an implementation prerequisite.","notes":"AUTHORITATIVE SCOPE SUPERSESSION (2026-07-13): this note overrides the stale v32-only clauses in the original description/design/AC. Exit condition for yla8.10 is: merged v35-compatible actuator; exact dry-run and receipted apply for only a7d004c9..., f19944c8..., fa0574f8...; those three reach byte_proven with all non-source-envelope state unchanged; reapply is idempotent; postflight proves those raw IDs no longer invalid. It is NOT an exit condition for yla8.10 to repair 282983b4..., 86298651..., or affadd9d..., nor to make the global byte-quarantined census zero: those three fail origin/parser equality and are exclusively owned by polylogue-lkrc. No v32 package/build/artifact or v32 rebuild is required or permitted for this closure.","status":"closed","priority":0,"issue_type":"bug","assignee":"Sinity","owner":"ezo.dev@gmail.com","created_at":"2026-07-12T18:45:47Z","created_by":"Sinity","updated_at":"2026-07-13T00:44:05Z","started_at":"2026-07-12T18:48:37Z","closed_at":"2026-07-13T00:44:05Z","close_reason":"Merged PR #2808 (3a5102b843) and source-v7 compatibility PR #2811 (c1d3c1fbc). Live stopped-daemon postflight repaired exactly a7d004c9..., f19944c8..., fa0574f8... under aggregate proof 8735245c... with verified 53.1GB blob/durable backup at /realm/staging/polylogue-sqlite/recovery/yla8-10-authority-20260713/polylogue-archive-20260713T003259Z. Receipt source-authority-repair.jsonl is planned→applied and names exactly those three. Backup comparison: source quick_check ok, FK0, relevant counts equal, all non-target raw rows identical, each target changed only logical_source_key/revision_kind/source_revision/baseline_raw_id/acquisition_generation/revision_authority. Reapply repaired=0 and receipt stayed 2 lines. Daemon restarted stable PID 2241036 NRestarts=0; Drive catch-up 0 errors; repaired cohort invalid=0. Remaining three unknown-export origin mismatches are explicitly excluded and tracked P0 polylogue-lkrc.","labels":["area:daemon","area:storage","area:test","delivery:A-trust-floor","horizon:frontier","lane:operational-resilience","spine"],"dependency_count":0,"dependent_count":1,"comment_count":0} diff --git a/docs/daemon.md b/docs/daemon.md index 2d915d13ec..e68a8e3e71 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -21,6 +21,13 @@ Check status: polylogued status ``` +Raw-evidence authority is an ordinary daemon invariant. After bounded raw +materialization, the daemon records one complete accepted-frontier census, +applies only byte/provenance-safe plans, and leaves conflicts or missing bytes +as durable remediation references in status. Operators can inspect the same +ledger with `polylogue ops maintenance raw-authority-frontier`; its apply +options are break-glass controls for exact plan IDs, not routine maintenance. + ## Auto-Discovery The daemon watches these directories by default: diff --git a/docs/plans/topology-target.yaml b/docs/plans/topology-target.yaml index ac875b6ace..f7f9c97421 100644 --- a/docs/plans/topology-target.yaml +++ b/docs/plans/topology-target.yaml @@ -857,7 +857,7 @@ files: target: polylogue/cli/commands/judge.py owner: stable - path: polylogue/cli/commands/maintenance/__init__.py - loc: 176 + loc: 146 target: polylogue/cli/commands/maintenance/__init__.py owner: stable - path: polylogue/cli/commands/maintenance/_archive_plan.py @@ -905,7 +905,7 @@ files: target: polylogue/cli/commands/maintenance/_preview.py owner: stable - path: polylogue/cli/commands/maintenance/_raw_identity.py - loc: 694 + loc: 220 target: polylogue/cli/commands/maintenance/_raw_identity.py owner: stable - path: polylogue/cli/commands/maintenance/_rebuild_index.py @@ -1418,7 +1418,7 @@ files: target: polylogue/daemon/catchup_status.py owner: stable - path: polylogue/daemon/cli.py - loc: 1983 + loc: 2018 target: polylogue/daemon/cli.py owner: stable - path: polylogue/daemon/compare.py @@ -1579,7 +1579,7 @@ files: target: polylogue/daemon/similarity.py owner: stable - path: polylogue/daemon/status.py - loc: 2670 + loc: 2677 target: polylogue/daemon/status.py owner: stable - path: polylogue/daemon/status_snapshot.py @@ -1659,7 +1659,7 @@ files: target: polylogue/daemon/workspace_routes.py owner: stable - path: polylogue/daemon/write_coordinator.py - loc: 428 + loc: 439 target: polylogue/daemon/write_coordinator.py owner: stable - path: polylogue/demo/__init__.py @@ -1945,7 +1945,7 @@ files: target: polylogue/maintenance/models.py owner: stable - path: polylogue/maintenance/offline_guard.py - loc: 47 + loc: 54 target: polylogue/maintenance/offline_guard.py owner: stable - path: polylogue/maintenance/planner.py @@ -3240,7 +3240,7 @@ files: target: TBD owner: storage-domain - path: polylogue/storage/archive_readiness.py - loc: 749 + loc: 818 target: TBD owner: storage-domain - path: polylogue/storage/archive_views.py @@ -3521,7 +3521,11 @@ files: target: polylogue/storage/raw/models.py owner: stable - path: polylogue/storage/raw_authority.py - loc: 1537 + loc: 1633 + target: TBD + owner: storage-domain + - path: polylogue/storage/raw_reconciler.py + loc: 1411 target: TBD owner: storage-domain - path: polylogue/storage/raw_retention.py @@ -3530,7 +3534,7 @@ files: owner: storage-root reason: storage-root cross-cutting helper - path: polylogue/storage/repair.py - loc: 7337 + loc: 6288 target: polylogue/storage/repair.py owner: storage-root reason: storage-root cross-cutting helper diff --git a/docs/topology-status.md b/docs/topology-status.md index c307cbdc91..d8e0e49740 100644 --- a/docs/topology-status.md +++ b/docs/topology-status.md @@ -31,9 +31,9 @@ Generated by `devtools render topology-status`. Reads `docs/plans/topology-targe - **Stable** (no move scoped): 843 - **Kernel** (polylogue/ root): 7 - **Primitives** (storage-root): 18 -- **TBD** (cell needs explicit assignment): 8 -- **Total declared**: 1006 -- **Realized polylogue/**/*.py**: 1006 files declared +- **TBD** (cell needs explicit assignment): 9 +- **Total declared**: 1007 +- **Realized polylogue/**/*.py**: 1007 files declared ### TBD cells (require explicit routing) @@ -46,5 +46,6 @@ These rows in the projection have no resolved target yet. Each needs an explicit - `polylogue/storage/block_anchor.py` — no rule yet - `polylogue/storage/index_generation.py` — no rule yet - `polylogue/storage/raw_authority.py` — no rule yet +- `polylogue/storage/raw_reconciler.py` — no rule yet - `polylogue/storage/table_existence.py` — no rule yet diff --git a/polylogue/cli/commands/maintenance/__init__.py b/polylogue/cli/commands/maintenance/__init__.py index 1e81f5dcd6..9c01d91d57 100644 --- a/polylogue/cli/commands/maintenance/__init__.py +++ b/polylogue/cli/commands/maintenance/__init__.py @@ -53,6 +53,12 @@ "rebuild_index_command", "Inspect or execute an authority-safe source-to-index rebuild.", ), + ( + "raw-authority-frontier", + "_raw_identity", + "raw_authority_frontier_command", + "Inspect the complete raw-authority frontier; apply is break-glass only.", + ), ( "raw-authority-census", "_raw_identity", @@ -71,42 +77,6 @@ "raw_authority_blocker_resolve_command", "Resolve one stale-plan blocker against current source evidence.", ), - ( - "missing-raw-blob-cursors", - "_raw_identity", - "missing_raw_blob_cursors_command", - "Invalidate cursors hiding missing raw-blob re-acquisition debt.", - ), - ( - "quarantined-accepted-raws", - "_raw_identity", - "quarantined_accepted_raws_command", - "Repair a typed accepted full raw whose byte authority stayed quarantined.", - ), - ( - "browser-capture-origin-mismatches", - "_raw_identity", - "browser_capture_origin_mismatches_command", - "Copy mismatched browser captures forward under parsed origin authority.", - ), - ( - "legacy-browser-capture-missing-native-id", - "_raw_identity", - "legacy_browser_capture_missing_native_id_command", - "Copy forward the narrow legacy browser shape whose native ID is NULL.", - ), - ( - "browser-canonical-authority-conflicts", - "_raw_identity", - "browser_canonical_authority_conflicts_command", - "Show why a byte-proven-rekey actuator refuses these browser-capture raws.", - ), - ( - "duplicate-raw-identity", - "_raw_identity", - "duplicate_raw_identity_command", - "Reconcile a pre-#2729 duplicate raw pair onto its post-fix accepted head.", - ), ("preview", "_preview", "preview_command", "Staleness inventory by model and scope. Read-only."), ("blob-gc", "_blob_gc", "blob_gc_command", "Preview or run lease-safe blob garbage collection."), ( diff --git a/polylogue/cli/commands/maintenance/_raw_identity.py b/polylogue/cli/commands/maintenance/_raw_identity.py index 703099256a..fbd8ef08fc 100644 --- a/polylogue/cli/commands/maintenance/_raw_identity.py +++ b/polylogue/cli/commands/maintenance/_raw_identity.py @@ -2,21 +2,78 @@ from __future__ import annotations -import contextlib import json -import sqlite3 -from dataclasses import asdict -from pathlib import Path -from typing import Any import click -from polylogue.archive.raw_materialization import source_path_native_id_candidates from polylogue.cli.shared.types import AppEnv from polylogue.config import Config from polylogue.paths import archive_root, render_root +@click.command("raw-authority-frontier") +@click.option("--apply-plan", "plan_ids", multiple=True, help="Exact immutable plan id; repeatable.") +@click.option("--preview-census", default=None, help="Completed dry-run census authorizing --apply-plan.") +@click.option("--yes", "confirmed", is_flag=True, help="Confirm the selected break-glass application.") +@click.option( + "--output-format", + type=click.Choice(["plain", "json"]), + default="plain", + show_default=True, +) +@click.pass_obj +def raw_authority_frontier_command( + env: AppEnv, + plan_ids: tuple[str, ...], + preview_census: str | None, + confirmed: bool, + output_format: str, +) -> None: + """Inspect the complete frontier or apply exact plans as break-glass work.""" + del env + from polylogue.storage.raw_reconciler import ( + apply_raw_authority_frontier, + inspect_raw_authority_frontier, + ) + + root = archive_root() + config = Config(archive_root=root, render_root=render_root(), sources=[]) + try: + if plan_ids: + if not confirmed: + raise click.ClickException("refusing raw-authority application without --yes") + if preview_census is None: + raise click.ClickException("--apply-plan requires --preview-census") + payload = apply_raw_authority_frontier( + config, + preview_census_id=preview_census, + selected_plan_ids=plan_ids, + ).to_dict() + else: + if preview_census is not None or confirmed: + raise click.ClickException("apply options require at least one --apply-plan") + payload = inspect_raw_authority_frontier(config).to_dict() + except (FileNotFoundError, KeyError, RuntimeError, ValueError) as exc: + if isinstance(exc, click.ClickException): + raise + raise click.ClickException(str(exc)) from exc + if output_format == "json": + click.echo(json.dumps(payload, indent=2, sort_keys=True)) + return + if plan_ids: + click.echo( + f"Applied {payload['executed_plan_count']}/{payload['selected_plan_count']} plan(s); " + f"retryable={payload['retryable_plan_count']} census={payload['census_id']}" + ) + return + click.echo( + f"Frontier {payload['census_id']}: accepted={payload['accepted_head_count']} " + f"plans={payload['plan_count']} executable={payload['executable_plan_count']}" + ) + click.echo(f"States: {json.dumps(payload['state_counts'], sort_keys=True)}") + click.echo(f"Details: {payload['query_handle']}") + + @click.command("raw-authority-census") @click.argument("query_handle") @click.option("--limit", type=click.IntRange(1, 500), default=100, show_default=True) @@ -109,6 +166,17 @@ def raw_authority_detail_command( @click.command("raw-authority-blocker-resolve") @click.option("--blocker-id", required=True, help="Exact unresolved durable blocker identifier.") @click.option("--reason", required=True, help="Operator rationale recorded in the immutable resolution receipt.") +@click.option( + "--assertion-id", + default=None, + help="Accepted judgment assertion required by a conflicting-authority blocker.", +) +@click.option( + "--judgment-disposition", + type=click.Choice(["retain_canonical_authority"]), + default=None, + help="Typed authority choice required when resolving a conflicting frontier.", +) @click.option("--yes", "confirmed", is_flag=True, help="Confirm resolving this blocker against current evidence.") @click.option( "--output-format", @@ -122,6 +190,8 @@ def raw_authority_blocker_resolve_command( env: AppEnv, blocker_id: str, reason: str, + assertion_id: str | None, + judgment_disposition: str | None, confirmed: bool, output_format: str, ) -> None: @@ -132,7 +202,13 @@ def raw_authority_blocker_resolve_command( from polylogue.storage.raw_authority import resolve_raw_authority_blocker try: - receipt = resolve_raw_authority_blocker(archive_root(), blocker_id, resolution=reason) + receipt = resolve_raw_authority_blocker( + archive_root(), + blocker_id, + resolution=reason, + assertion_id=assertion_id, + judgment_disposition=judgment_disposition, + ) except (FileNotFoundError, KeyError, RuntimeError, ValueError) as exc: raise click.ClickException(str(exc)) from exc if output_format == "json": @@ -142,553 +218,3 @@ def raw_authority_blocker_resolve_command( current_plan = receipt.get("current_plan") if isinstance(current_plan, dict): click.echo(f"Current plan: {current_plan.get('plan_id', 'unknown')}") - - -def _raw_blob_path_for_hash(root: Path, blob_hash: bytes | str) -> Path | None: - hex_hash = blob_hash.hex() if isinstance(blob_hash, bytes) else str(blob_hash).lower() - if len(hex_hash) != 64 or any(char not in "0123456789abcdef" for char in hex_hash): - return None - return root / "blob" / hex_hash[:2] / hex_hash[2:] - - -def _missing_raw_blob_cursor_candidates(root: Path, *, limit: int | None = None) -> list[dict[str, object]]: - source_db = root / "source.db" - index_db = root / "index.db" - ops_db = root / "ops.db" - if not source_db.exists() or not index_db.exists() or not ops_db.exists(): - return [] - conn = sqlite3.connect(f"file:{source_db}?mode=ro", uri=True) - conn.row_factory = sqlite3.Row - ops_conn = sqlite3.connect(f"file:{ops_db}?mode=ro", uri=True) - ops_conn.row_factory = sqlite3.Row - try: - conn.execute("ATTACH DATABASE ? AS index_tier", (str(index_db),)) - rows = conn.execute( - """ - SELECT - r.raw_id, - r.origin, - r.native_id, - r.source_path, - r.blob_hash, - r.blob_size, - r.validation_status, - r.parse_error - FROM raw_sessions AS r - LEFT JOIN index_tier.sessions AS s_by_raw ON s_by_raw.raw_id = r.raw_id - LEFT JOIN index_tier.sessions AS s_by_native - ON r.native_id IS NOT NULL - AND s_by_native.origin = r.origin - AND s_by_native.native_id = r.native_id - WHERE r.blob_hash IS NOT NULL - AND r.source_path IS NOT NULL - AND s_by_raw.raw_id IS NULL - AND s_by_native.native_id IS NULL - AND NOT ( - r.validation_status = 'skipped' - AND r.parsed_at_ms IS NOT NULL - AND r.parse_error IS NULL - ) - ORDER BY r.origin, r.blob_size DESC, r.raw_id - """ - ).fetchall() - candidates: list[dict[str, object]] = [] - seen_paths: set[str] = set() - for row in rows: - source_path = str(row["source_path"] or "") - if not source_path or source_path in seen_paths: - continue - blob_path = _raw_blob_path_for_hash(root, row["blob_hash"]) - if blob_path is None or blob_path.exists() or not Path(source_path).exists(): - continue - if _raw_materialized_by_source_path_candidate(conn, row): - continue - cursor = ops_conn.execute( - "SELECT stat_size, byte_offset, updated_at_ms FROM ingest_cursor WHERE source_path = ?", - (source_path,), - ).fetchone() - if cursor is None: - continue - candidates.append( - { - "source_path": source_path, - "raw_id": str(row["raw_id"]), - "origin": str(row["origin"] or ""), - "native_id": str(row["native_id"] or ""), - "blob_path": str(blob_path), - "blob_size": int(row["blob_size"] or 0), - "cursor_stat_size": int(cursor["stat_size"] or 0), - "cursor_byte_offset": int(cursor["byte_offset"] or 0), - "cursor_updated_at_ms": int(cursor["updated_at_ms"] or 0), - } - ) - seen_paths.add(source_path) - if limit is not None and len(candidates) >= limit: - break - return candidates - finally: - ops_conn.close() - conn.close() - - -def _raw_materialized_by_source_path_candidate(conn: sqlite3.Connection, row: sqlite3.Row) -> bool: - origin = str(row["origin"] or "") - if not origin: - return False - for native_id in source_path_native_id_candidates(str(row["source_path"] or "")): - existing = conn.execute( - """ - SELECT 1 - FROM index_tier.sessions - WHERE origin = ? - AND native_id = ? - LIMIT 1 - """, - (origin, native_id), - ).fetchone() - if existing is not None: - return True - return False - - -@click.command("missing-raw-blob-cursors") -@click.option("--apply", "apply_changes", is_flag=True, help="Delete matching rebuildable live cursor rows.") -@click.option("--limit", "-l", type=int, default=None, help="Limit the number of candidate source paths.") -@click.option( - "--output-format", - "output_format", - type=click.Choice(["plain", "json"]), - default="plain", - show_default=True, - help="Output format.", -) -@click.pass_obj -def missing_raw_blob_cursors_command( - env: AppEnv, - apply_changes: bool, - limit: int | None, - output_format: str, -) -> None: - """Invalidate cursors hiding missing raw-blob re-acquisition debt. - - This command only touches ``ops.db.ingest_cursor`` rows. It leaves - source-tier raw rows, source files, blobs, index rows, and user state - intact so the next daemon catch-up can re-acquire through the normal - ingestion path. - """ - del env - root = archive_root() - candidates = _missing_raw_blob_cursor_candidates(root, limit=limit) - deleted = 0 - if apply_changes and candidates: - ops_db = root / "ops.db" - with contextlib.closing(sqlite3.connect(ops_db)) as conn: - for candidate in candidates: - deleted += conn.execute( - "DELETE FROM ingest_cursor WHERE source_path = ?", - (str(candidate["source_path"]),), - ).rowcount - conn.commit() - - payload = { - "archive_root": str(root), - "mode": "apply" if apply_changes else "dry-run", - "candidate_count": len(candidates), - "deleted_cursor_count": deleted, - "candidates": candidates, - "next_action": "restart or run polylogued catch-up" if apply_changes and deleted else None, - } - if output_format == "json": - click.echo(json.dumps(payload, indent=2, sort_keys=True)) - return - - action = "Deleted" if apply_changes else "Would delete" - click.echo(f"{action} {deleted if apply_changes else len(candidates)} live cursor row(s)") - for candidate in candidates[:10]: - click.echo( - f" {candidate['origin']} {candidate['source_path']} " - f"raw={candidate['raw_id']} blob_size={candidate['blob_size']}" - ) - if len(candidates) > 10: - click.echo(f" ... {len(candidates) - 10} more") - if apply_changes and deleted: - click.echo("Next: restart or run polylogued catch-up.") - - -@click.command("quarantined-accepted-raws") -@click.option("--raw-id", "raw_ids", multiple=True, required=True, help="Exact retained raw SHA-256 id (repeatable).") -@click.option("--apply", "apply_changes", is_flag=True, help="Apply only after every target passes exact proof.") -@click.option("--proof-digest", help="Exact aggregate digest emitted by the matching dry-run.") -@click.option( - "--receipt", - "receipt_path", - type=click.Path(path_type=Path, dir_okay=False), - help="Required append-only operator recovery receipt path for --apply.", -) -@click.option( - "--output-format", - "output_format", - type=click.Choice(["plain", "json"]), - default="plain", - show_default=True, -) -@click.pass_obj -def quarantined_accepted_raws_command( - env: AppEnv, - raw_ids: tuple[str, ...], - apply_changes: bool, - proof_digest: str | None, - receipt_path: Path | None, - output_format: str, -) -> None: - """Repair a typed accepted full raw whose byte authority stayed quarantined. - - The actuator revalidates the existing immutable selected-baseline receipt, - retained blob bytes, parser-normalized session identity/content, and the - current accepted head. It never changes the index head or its receipt. - Apply additionally writes an exclusive, fsynced planned→applied operator - receipt so the source-only refinement is crash-resumable and auditable. - """ - del env - if apply_changes and receipt_path is None: - raise click.UsageError("--apply requires --receipt PATH") - if apply_changes and proof_digest is None: - raise click.UsageError("--apply requires --proof-digest from the exact dry-run") - root = archive_root() - config = Config(archive_root=root, render_root=render_root(), sources=[], db_path=root / "index.db") - from polylogue.storage.repair import repair_quarantined_accepted_raws - - try: - report = repair_quarantined_accepted_raws( - config, - list(raw_ids), - apply=apply_changes, - receipt_path=receipt_path, - proof_digest=proof_digest, - ) - except (RuntimeError, ValueError) as exc: - raise click.ClickException(str(exc)) from exc - payload = asdict(report) - if output_format == "json": - click.echo(json.dumps(payload, indent=2, sort_keys=True)) - return - click.echo( - f"{report.mode}: requested={report.requested_count} eligible={report.eligible_count} " - f"already_repaired={report.already_repaired_count} repaired={report.repaired_count} " - f"ineligible={report.ineligible_count}" - ) - click.echo(f"Proof digest: {report.proof_digest}") - for item in report.items: - click.echo(f" {item.raw_id} {item.status} proof={item.proof_digest or 'unavailable'}: {item.reason}") - if report.receipt_path is not None: - click.echo(f"Receipt: {report.receipt_path}") - - -@click.command("browser-capture-origin-mismatches") -@click.option("--raw-id", "raw_ids", multiple=True, required=True, help="Exact mismatched raw id (repeatable).") -@click.option("--apply", "apply_changes", is_flag=True, help="Copy forward only after every target passes proof.") -@click.option("--proof-digest", help="Exact aggregate digest emitted by the matching dry-run.") -@click.option( - "--receipt", - "receipt_path", - type=click.Path(path_type=Path, dir_okay=False), - help="Required append-only operator recovery receipt path for --apply.", -) -@click.option( - "--output-format", - "output_format", - type=click.Choice(["plain", "json"]), - default="plain", - show_default=True, -) -@click.pass_obj -def browser_capture_origin_mismatches_command( - env: AppEnv, - raw_ids: tuple[str, ...], - apply_changes: bool, - proof_digest: str | None, - receipt_path: Path | None, - output_format: str, -) -> None: - """Copy mismatched browser captures forward under parsed origin authority. - - The old raw, blob, membership, byte head, and application receipts remain - immutable. Apply creates a new raw reference to the exact retained blob, - records a canonical head, and advances only the derived session raw pointer. - """ - del env - if apply_changes and receipt_path is None: - raise click.UsageError("--apply requires --receipt PATH") - if apply_changes and proof_digest is None: - raise click.UsageError("--apply requires --proof-digest from the exact dry-run") - root = archive_root() - config = Config(archive_root=root, render_root=render_root(), sources=[], db_path=root / "index.db") - from polylogue.storage.repair import repair_browser_capture_origin_mismatches - - try: - report = repair_browser_capture_origin_mismatches( - config, - list(raw_ids), - apply=apply_changes, - receipt_path=receipt_path, - proof_digest=proof_digest, - ) - except (RuntimeError, ValueError) as exc: - raise click.ClickException(str(exc)) from exc - payload = asdict(report) - if output_format == "json": - click.echo(json.dumps(payload, indent=2, sort_keys=True)) - return - click.echo( - f"{report.mode}: requested={report.requested_count} eligible={report.eligible_count} " - f"already_repaired={report.already_repaired_count} repaired={report.repaired_count} " - f"ineligible={report.ineligible_count}" - ) - click.echo(f"Proof digest: {report.proof_digest}") - for item in report.items: - click.echo( - f" {item.raw_id} {item.status} strategy={item.repair_strategy or 'unavailable'} " - f"replacement={item.replacement_raw_id or 'unavailable'} " - f"proof={item.proof_digest or 'unavailable'}: {item.reason}" - ) - if report.receipt_path is not None: - click.echo(f"Receipt: {report.receipt_path}") - - -@click.command("legacy-browser-capture-missing-native-id") -@click.option( - "--raw-id", "raw_ids", multiple=True, required=True, help="Exact legacy raw id with native_id NULL (repeatable)." -) -@click.option( - "--apply", "apply_changes", is_flag=True, help="Copy forward only after every legacy witness passes proof." -) -@click.option("--proof-digest", help="Exact aggregate digest emitted by the matching dry-run.") -@click.option( - "--receipt", - "receipt_path", - type=click.Path(path_type=Path, dir_okay=False), - help="Required append-only operator recovery receipt path for --apply.", -) -@click.option( - "--output-format", - "output_format", - type=click.Choice(["plain", "json"]), - default="plain", - show_default=True, -) -@click.pass_obj -def legacy_browser_capture_missing_native_id_command( - env: AppEnv, - raw_ids: tuple[str, ...], - apply_changes: bool, - proof_digest: str | None, - receipt_path: Path | None, - output_format: str, -) -> None: - """Copy forward the narrow legacy browser shape whose native ID is NULL. - - The original raw, blob, memberships, head, and application receipts remain - immutable. This is not an alternate mode of ordinary origin repair. - """ - del env - if apply_changes and receipt_path is None: - raise click.UsageError("--apply requires --receipt PATH") - if apply_changes and proof_digest is None: - raise click.UsageError("--apply requires --proof-digest from the exact dry-run") - root = archive_root() - config = Config(archive_root=root, render_root=render_root(), sources=[], db_path=root / "index.db") - from polylogue.storage.repair import repair_legacy_browser_capture_missing_native_ids - - try: - report = repair_legacy_browser_capture_missing_native_ids( - config, - list(raw_ids), - apply=apply_changes, - receipt_path=receipt_path, - proof_digest=proof_digest, - ) - except (RuntimeError, ValueError) as exc: - raise click.ClickException(str(exc)) from exc - payload = asdict(report) - if output_format == "json": - click.echo(json.dumps(payload, indent=2, sort_keys=True)) - return - click.echo( - f"{report.mode}: requested={report.requested_count} eligible={report.eligible_count} " - f"already_repaired={report.already_repaired_count} repaired={report.repaired_count} " - f"ineligible={report.ineligible_count}" - ) - click.echo(f"Proof digest: {report.proof_digest}") - for item in report.items: - click.echo( - f" {item.raw_id} {item.status} parsed_native_id={item.parser_derived_native_id or 'unavailable'} " - f"proof={item.proof_digest or 'unavailable'}: {item.reason}" - ) - if report.receipt_path is not None: - click.echo(f"Receipt: {report.receipt_path}") - - -@click.command("browser-canonical-authority-conflicts") -@click.option( - "--raw-id", - "raw_ids", - multiple=True, - required=True, - help="Exact unknown-export raw id a safe rekey refuses (repeatable).", -) -@click.option( - "--record", - "record_blockers", - is_flag=True, - help="Persist each conflict as a durable, non-injected user.db blocker candidate.", -) -@click.option( - "--output-format", - "output_format", - type=click.Choice(["plain", "json"]), - default="plain", - show_default=True, -) -@click.pass_obj -def browser_canonical_authority_conflicts_command( - env: AppEnv, - raw_ids: tuple[str, ...], - record_blockers: bool, - output_format: str, -) -> None: - """Show why a byte-proven-rekey actuator refuses these browser-capture raws. - - Read-only by default: re-runs the ordinary rekey actuator's exact - eligibility proof and, for every raw that stays ineligible, re-derives the - competing-authority evidence it discards on its own reject path (competing - head content hash/frontier kind/decision, any blocking membership row, and - -- when both sides are single-session byte-frontier raws -- the first - diverging message index). Never selects an authority between the two - histories. Pass ``--record`` to additionally persist each conflict as one - ``AssertionKind.BLOCKER`` candidate assertion in ``user.db`` (always - ``status=candidate``/``inject:false``; an operator must judge it - explicitly -- see ``polylogue mark`` / assertion judgment tooling). - """ - del env - root = archive_root() - config = Config(archive_root=root, render_root=render_root(), sources=[], db_path=root / "index.db") - from polylogue.storage.repair import ( - inspect_browser_canonical_authority_conflicts, - record_browser_canonical_authority_conflict_blockers, - ) - - assertion_ids: tuple[str, ...] = () - try: - if record_blockers: - report, assertion_ids = record_browser_canonical_authority_conflict_blockers(config, list(raw_ids)) - else: - report = inspect_browser_canonical_authority_conflicts(config, list(raw_ids)) - except (RuntimeError, ValueError) as exc: - raise click.ClickException(str(exc)) from exc - payload: dict[str, Any] = asdict(report) - if record_blockers: - payload["assertion_ids"] = list(assertion_ids) - if output_format == "json": - click.echo(json.dumps(payload, indent=2, sort_keys=True)) - return - click.echo(f"requested={report.requested_count} conflicts={report.conflict_count} resolved={report.resolved_count}") - for item in report.items: - click.echo( - f" {item.raw_id} competing={item.competing_raw_id or 'none'} " - f"frontier={item.competing_frontier_kind or 'unavailable'} " - f"divergent_message_index={item.divergent_message_index if item.divergent_message_index is not None else 'unavailable'}: " - f"{item.divergence_note or item.reason}" - ) - if record_blockers: - for assertion_id in assertion_ids: - click.echo(f"Blocker: {assertion_id}") - - -@click.command("duplicate-raw-identity") -@click.option( - "--pair", - "pairs", - multiple=True, - required=True, - metavar="STALE_RAW_ID:CANONICAL_RAW_ID", - help="Stale (currently accepted) and canonical (post-fix, dangling) raw id pair (repeatable).", -) -@click.option("--apply", "apply_changes", is_flag=True, help="Apply only after every pair passes exact proof.") -@click.option("--proof-digest", help="Exact aggregate digest emitted by the matching dry-run.") -@click.option( - "--receipt", - "receipt_path", - type=click.Path(path_type=Path, dir_okay=False), - help="Required append-only operator recovery receipt path for --apply.", -) -@click.option( - "--output-format", - "output_format", - type=click.Choice(["plain", "json"]), - default="plain", - show_default=True, -) -@click.pass_obj -def duplicate_raw_identity_command( - env: AppEnv, - pairs: tuple[str, ...], - apply_changes: bool, - proof_digest: str | None, - receipt_path: Path | None, - output_format: str, -) -> None: - """Reconcile a pre-#2729 duplicate raw pair onto its post-fix accepted head. - - PR #2729 aligned new ingests on one deterministic raw-id scheme, but did - not retroactively repair raw pairs that already duplicated under the OLD - scheme: the accepted head stays bound to the stale raw while its - post-fix twin sits orphaned. This actuator proves both raws are - byte-identical retained duplicates of one logical session, then - repoints the accepted head and session pointer to the canonical raw via - the existing revision-application machinery. The stale raw's own row is - never mutated or deleted. Apply additionally writes an exclusive, - fsynced planned-then-applied operator receipt so the repair is - crash-resumable and auditable. - """ - del env - if apply_changes and receipt_path is None: - raise click.UsageError("--apply requires --receipt PATH") - if apply_changes and proof_digest is None: - raise click.UsageError("--apply requires --proof-digest from the exact dry-run") - parsed_pairs: list[tuple[str, str]] = [] - for pair in pairs: - stale_raw_id, sep, canonical_raw_id = pair.partition(":") - if not sep: - raise click.UsageError(f"--pair must be STALE_RAW_ID:CANONICAL_RAW_ID, got {pair!r}") - parsed_pairs.append((stale_raw_id, canonical_raw_id)) - root = archive_root() - config = Config(archive_root=root, render_root=render_root(), sources=[], db_path=root / "index.db") - from polylogue.storage.repair import repair_duplicate_raw_identity - - try: - report = repair_duplicate_raw_identity( - config, - parsed_pairs, - apply=apply_changes, - receipt_path=receipt_path, - proof_digest=proof_digest, - ) - except (RuntimeError, ValueError) as exc: - raise click.ClickException(str(exc)) from exc - payload = asdict(report) - if output_format == "json": - click.echo(json.dumps(payload, indent=2, sort_keys=True)) - return - click.echo( - f"{report.mode}: requested={report.requested_count} eligible={report.eligible_count} " - f"already_repaired={report.already_repaired_count} repaired={report.repaired_count} " - f"ineligible={report.ineligible_count}" - ) - click.echo(f"Proof digest: {report.proof_digest}") - for item in report.items: - click.echo( - f" {item.stale_raw_id}->{item.canonical_raw_id} {item.status} " - f"proof={item.proof_digest or 'unavailable'}: {item.reason}" - ) - if report.receipt_path is not None: - click.echo(f"Receipt: {report.receipt_path}") diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 93f1554b5b..9796eab281 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -626,14 +626,49 @@ def _drain_raw_materialization_once(*, limit: int = _RAW_MATERIALIZATION_CONVERG render_root=render_root(), sources=[], ) + from polylogue.storage.raw_reconciler import recover_interrupted_raw_authority_frontier + + recover_interrupted_raw_authority_frontier(config) try: result = repair_raw_materialization(config, dry_run=False, raw_artifact_limit=limit) finally: _close_raw_materialization_fts(config.archive_root / "index.db") _emit_raw_materialization_pass(result) + frontier_repaired = _converge_raw_authority_frontier(config, limit=min(limit, 8)) if not result.success: logger.warning("raw materialization: bounded convergence incomplete: %s", result.detail) - return result.repaired_count + return result.repaired_count + frontier_repaired + + +def _converge_raw_authority_frontier(config: Any, *, limit: int) -> int: + """Census the entire accepted frontier and execute a bounded safe slice. + + This runs only beneath ``DaemonWriteCoordinator``. Conflicts, missing + bytes, unresolved provenance, and corruption are persisted as obligations; + only strategies carrying an exact deterministic proof become selectable. + """ + from polylogue.storage.raw_reconciler import ( + apply_raw_authority_frontier, + inspect_raw_authority_frontier, + ) + + census = inspect_raw_authority_frontier(config) + executable = tuple(item.plan_id for item in census.items if item.executable)[:limit] + if not executable: + return 0 + report = apply_raw_authority_frontier( + config, + preview_census_id=census.census_id, + selected_plan_ids=executable, + ) + if report.retryable_plan_count: + logger.warning( + "raw authority: %d/%d selected frontier plans remain retryable; census=%s", + report.retryable_plan_count, + report.selected_plan_count, + report.census_id, + ) + return report.executed_plan_count def _emit_raw_materialization_pass(result: Any) -> None: diff --git a/polylogue/daemon/status.py b/polylogue/daemon/status.py index 0076a83da0..f02d4d1ae4 100644 --- a/polylogue/daemon/status.py +++ b/polylogue/daemon/status.py @@ -161,6 +161,13 @@ class RawMaterializationReadiness(BaseModel): category_counts: dict[str, int] = Field(default_factory=dict) source_family_counts: dict[str, int] = Field(default_factory=dict) sampled_rows: list[dict[str, object]] = Field(default_factory=list) + raw_authority_census: dict[str, object] | None = None + raw_authority_frontier: dict[str, object] | None = None + raw_authority_frontier_blocking_count: int = 0 + raw_authority_frontier_remediation_refs: list[dict[str, object]] = Field(default_factory=list) + raw_authority_blocker_count: int = 0 + raw_authority_pending_census_count: int = 0 + raw_authority_ledger_counts: dict[str, int] = Field(default_factory=dict) class RawFrontierIntegrity(BaseModel): diff --git a/polylogue/daemon/write_coordinator.py b/polylogue/daemon/write_coordinator.py index ddda3db251..feeb19fad5 100644 --- a/polylogue/daemon/write_coordinator.py +++ b/polylogue/daemon/write_coordinator.py @@ -76,6 +76,17 @@ class _WriteRequest: } +def daemon_write_lease_active() -> bool: + """Return whether the current context owns the daemon's writer gate. + + ``run_sync`` deliberately propagates context into its worker thread, so + storage code can distinguish a coordinator-authorized online write from an + unrelated maintenance process racing the daemon. This is an authority + check, not merely a daemon-process check. + """ + return _ACTIVE_LEASE.get() is not None + + class DaemonWriteCoordinator: """Fair async gate around every archive write actor in one daemon. diff --git a/polylogue/maintenance/offline_guard.py b/polylogue/maintenance/offline_guard.py index a614db7466..174a03b085 100644 --- a/polylogue/maintenance/offline_guard.py +++ b/polylogue/maintenance/offline_guard.py @@ -35,6 +35,13 @@ def offline_maintenance_block_reason( """Return a refusal reason when offline maintenance would race the daemon.""" if dry_run or not active: return None + # A daemon-owned writer is already serialized against every other archive + # mutation. Treat it as the online equivalent of the offline exclusion + # boundary instead of rejecting the daemon's own convergence work. + from polylogue.daemon.write_coordinator import daemon_write_lease_active + + if daemon_write_lease_active(): + return None daemon_pid = running_daemon_pid(config) if daemon_pid is None: return None diff --git a/polylogue/storage/archive_readiness.py b/polylogue/storage/archive_readiness.py index ec6467bc76..7518553de3 100644 --- a/polylogue/storage/archive_readiness.py +++ b/polylogue/storage/archive_readiness.py @@ -16,6 +16,7 @@ ) from polylogue.archive.revision_authority import BYTE_AUTHORITY_CENSUS_DETAIL from polylogue.logging import get_logger +from polylogue.storage.raw_authority import raw_authority_detail_query_handle logger = get_logger(__name__) @@ -91,6 +92,9 @@ def raw_materialization_ready(readiness: Mapping[str, Any] | object | None) -> b # required the classifier cannot be claimed when the classifier failed. if readiness.get("debt_classifier_error"): return False + frontier = readiness.get("raw_authority_frontier") + if not isinstance(frontier, Mapping) or frontier.get("lifecycle_status") != "completed": + return False blocking_keys = ( "critical", "warning", @@ -102,6 +106,8 @@ def raw_materialization_ready(readiness: Mapping[str, Any] | object | None) -> b "lost_source_evidence_count", "unchecked", "affected_unchecked", + "raw_authority_frontier_blocking_count", + "raw_authority_pending_census_count", ) return all(_read_int(readiness, key) == 0 for key in blocking_keys) @@ -234,6 +240,9 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob lost_source_evidence_count = _missing_source_raw_session_count(conn) lost_source_evidence_samples = _missing_source_raw_session_samples(conn) authority_census: dict[str, object] | None = None + authority_frontier: dict[str, object] | None = None + authority_frontier_blocking_count = 0 + authority_frontier_remediation_refs: list[dict[str, object]] = [] authority_pending_census_count = 0 if _table_columns(conn, "source", "raw_authority_censuses"): authority_pending_census_count = int( @@ -277,6 +286,56 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob "pending_census_count": authority_pending_census_count, "query_handle": (f"polylogue://raw-authority-census/{census_row['census_id']}/0"), } + frontier_row = conn.execute( + """ + SELECT census_id, sequence_no, inventory_digest, residual_digest, + plan_count, executable_plan_count, residual_plan_count, + lifecycle_status, completed_at_ms, scope_json, + post_residual_json + FROM source.raw_authority_censuses + WHERE lifecycle_status IN ('completed', 'interrupted') + AND json_extract(scope_json, '$.schema') = + 'polylogue.raw-authority-frontier-scope.v1' + ORDER BY sequence_no DESC LIMIT 1 + """ + ).fetchone() + if frontier_row is not None: + import json + + frontier_scope = json.loads(str(frontier_row["scope_json"])) + # An apply census records its pre-application scope for + # auditability, then publishes the actual frontier in the + # postflight residual. Readiness must reflect that + # terminal state rather than keep an already repaired plan + # blocking until some later inspection happens to run. + frontier_post_residual = json.loads(str(frontier_row["post_residual_json"] or "{}")) + postflight_state_counts = frontier_post_residual.get("state_counts") + scope_state_counts = frontier_scope.get("state_counts") + state_counts_source = ( + postflight_state_counts if isinstance(postflight_state_counts, Mapping) else scope_state_counts + ) + frontier_state_counts = { + str(key): int(value) for key, value in dict(state_counts_source or {}).items() + } + nonblocking_states = {"proven_current", "superseded"} + authority_frontier_blocking_count = sum( + count for state, count in frontier_state_counts.items() if state not in nonblocking_states + ) + authority_frontier = { + "census_id": str(frontier_row["census_id"]), + "sequence_no": int(frontier_row["sequence_no"]), + "inventory_digest": str(frontier_scope.get("inventory_digest") or ""), + "plan_inventory_digest": str(frontier_row["inventory_digest"]), + "residual_digest": str(frontier_row["residual_digest"]), + "plan_count": int(frontier_row["plan_count"]), + "executable_plan_count": int(frontier_row["executable_plan_count"]), + "residual_plan_count": int(frontier_row["residual_plan_count"]), + "state_counts": frontier_state_counts, + "blocking_count": authority_frontier_blocking_count, + "lifecycle_status": str(frontier_row["lifecycle_status"]), + "completed_at_ms": int(frontier_row["completed_at_ms"]), + "query_handle": (f"polylogue://raw-authority-census/{frontier_row['census_id']}/0"), + } authority_blocker_count = 0 if _table_columns(conn, "source", "raw_authority_blockers"): authority_blocker_count = int( @@ -284,6 +343,25 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob "SELECT COUNT(*) FROM source.raw_authority_blockers WHERE resolved_at_ms IS NULL" ).fetchone()[0] ) + authority_frontier_remediation_refs = [ + { + "blocker_id": str(blocker_id), + "plan_id": str(plan_id), + "detail_query_handle": raw_authority_detail_query_handle(str(census_id), str(plan_id)), + } + for blocker_id, plan_id, census_id in conn.execute( + """ + SELECT b.blocker_id, b.plan_id, b.census_id + FROM source.raw_authority_blockers AS b + JOIN source.raw_authority_plans AS p ON p.plan_id = b.plan_id + WHERE b.resolved_at_ms IS NULL + AND json_extract(p.authority_witness_json, '$.schema') = + 'polylogue.raw-authority-frontier-plan.v1' + ORDER BY b.created_at_ms, b.blocker_id + LIMIT 16 + """ + ) + ] except Exception as exc: return { "available": False, @@ -343,6 +421,9 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob "category_counts": category_counts, "source_family_counts": {str(item["origin"]): int(item["count"] or 0) for item in family_rows}, "raw_authority_census": authority_census, + "raw_authority_frontier": authority_frontier, + "raw_authority_frontier_blocking_count": authority_frontier_blocking_count, + "raw_authority_frontier_remediation_refs": authority_frontier_remediation_refs, "raw_authority_blocker_count": authority_blocker_count, "raw_authority_pending_census_count": authority_pending_census_count, "raw_authority_ledger_counts": { diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index e2d771bca2..66de0075a1 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -692,6 +692,36 @@ def unresolved_raw_authority_blockers(archive_root: Path) -> int: ) +def unresolved_raw_replay_blockers(archive_root: Path) -> int: + """Count stale/application blockers that must stop ordinary raw replay. + + Frontier obligations are durable and readiness-blocking, but one missing or + conflicting authority must not starve unrelated, independently proven raw + components. + """ + source_db = archive_root / "source.db" + if not source_db.is_file(): + return 0 + with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as conn: + exists = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='raw_authority_blockers'" + ).fetchone() + if exists is None: + return 0 + return int( + conn.execute( + """ + SELECT COUNT(*) + FROM raw_authority_blockers AS b + JOIN raw_authority_plans AS p ON p.plan_id = b.plan_id + WHERE b.resolved_at_ms IS NULL + AND COALESCE(json_extract(p.authority_witness_json, '$.schema'), '') != + 'polylogue.raw-authority-frontier-plan.v1' + """ + ).fetchone()[0] + ) + + def record_raw_authority_census( archive_root: Path, plans: Sequence[RawReplayPlan], @@ -1169,16 +1199,33 @@ def finalize_raw_authority_census( if pending: raise RuntimeError(f"raw authority census still has {pending} pending selected outcome(s)") post_ids = {plan.plan_id for plan in post_plans} - persistent = { - str(row[0]) - for row in conn.execute( + selected_inputs = { + str(raw_id) + for (input_raw_ids_json,) in conn.execute( """ - SELECT plan_id FROM raw_authority_census_plans - WHERE census_id = ? AND outcome_status IN ('retryable', 'carried_forward') + SELECT p.input_raw_ids_json + FROM raw_authority_census_plans AS cp + JOIN raw_authority_plans AS p ON p.plan_id = cp.plan_id + WHERE cp.census_id = ? AND cp.selected = 1 """, (census_id,), ) + for raw_id in json.loads(str(input_raw_ids_json)) } + persistent: set[str] = set() + for plan_id, outcome_status, input_raw_ids_json in conn.execute( + """ + SELECT cp.plan_id, cp.outcome_status, p.input_raw_ids_json + FROM raw_authority_census_plans AS cp + JOIN raw_authority_plans AS p ON p.plan_id = cp.plan_id + WHERE cp.census_id = ? + AND cp.outcome_status IN ('retryable', 'carried_forward') + """, + (census_id,), + ): + plan_inputs = {str(raw_id) for raw_id in json.loads(str(input_raw_ids_json))} + if str(outcome_status) == RawReplayPlanStatus.RETRYABLE.value or plan_inputs.isdisjoint(selected_inputs): + persistent.add(str(plan_id)) if not persistent.issubset(post_ids): raise RuntimeError( f"raw authority postflight changed a retryable/carried-forward plan: {sorted(persistent - post_ids)}" @@ -1265,6 +1312,8 @@ def recover_interrupted_raw_authority_censuses( JOIN raw_authority_plans AS p ON p.plan_id = cp.plan_id WHERE c.lifecycle_status = 'planned' AND cp.selected = 1 AND cp.outcome_recorded = 0 + AND COALESCE(json_extract(p.authority_witness_json, '$.schema'), '') != + 'polylogue.raw-authority-frontier-plan.v1' ORDER BY c.sequence_no, cp.ordinal """ ).fetchall() @@ -1275,6 +1324,15 @@ def recover_interrupted_raw_authority_censuses( SELECT census_id, scope_json FROM raw_authority_censuses WHERE lifecycle_status = 'planned' + AND EXISTS ( + SELECT 1 + FROM raw_authority_census_plans AS cp + JOIN raw_authority_plans AS p ON p.plan_id = cp.plan_id + WHERE cp.census_id = raw_authority_censuses.census_id + AND cp.selected = 1 AND cp.outcome_recorded = 0 + AND COALESCE(json_extract(p.authority_witness_json, '$.schema'), '') != + 'polylogue.raw-authority-frontier-plan.v1' + ) ORDER BY sequence_no """ ) @@ -1311,7 +1369,14 @@ def recover_interrupted_raw_authority_censuses( return census_scopes -def resolve_raw_authority_blocker(archive_root: Path, blocker_id: str, *, resolution: str) -> JSONDocument: +def resolve_raw_authority_blocker( + archive_root: Path, + blocker_id: str, + *, + resolution: str, + assertion_id: str | None = None, + judgment_disposition: str | None = None, +) -> JSONDocument: """Explicitly acknowledge current evidence and reopen replanning.""" if not resolution.strip(): raise ValueError("raw authority blocker resolution must be non-empty") @@ -1322,7 +1387,10 @@ def resolve_raw_authority_blocker(archive_root: Path, blocker_id: str, *, resolu conn.execute("BEGIN IMMEDIATE") row = conn.execute( """ - SELECT b.blocker_id, b.plan_id, b.census_id, b.expected_json, p.input_raw_ids_json + SELECT b.blocker_id, b.plan_id, b.census_id, b.expected_json, + b.observed_json, p.input_raw_ids_json, p.input_digest, + p.logical_keys_json, p.authority_witness_json, + p.source_preconditions_json, p.index_preconditions_json FROM raw_authority_blockers AS b JOIN raw_authority_plans AS p ON p.plan_id = b.plan_id WHERE b.blocker_id = ? AND b.resolved_at_ms IS NULL @@ -1332,8 +1400,31 @@ def resolve_raw_authority_blocker(archive_root: Path, blocker_id: str, *, resolu if row is None: conn.rollback() raise KeyError(blocker_id) - input_raw_ids = tuple(str(value) for value in json.loads(str(row["input_raw_ids_json"]))) - observed = build_raw_replay_plan(conn, input_raw_ids) + stored_plan = _raw_replay_plan_from_row(row) + witness_schema = stored_plan.authority_witness.get("schema") + frontier_observed = json.loads(str(row["observed_json"])) + if witness_schema == "polylogue.raw-authority-frontier-plan.v1": + expected_assertion_id = frontier_observed.get("judgment_assertion_id") + if expected_assertion_id is not None: + if assertion_id != expected_assertion_id: + conn.rollback() + raise RuntimeError("frontier judgment blocker requires its exact accepted assertion id") + user_db = archive_root / "user.db" + with closing(sqlite3.connect(f"file:{user_db}?mode=ro", uri=True)) as user_conn: + assertion = user_conn.execute( + "SELECT status FROM assertions WHERE assertion_id = ?", + (assertion_id,), + ).fetchone() + if assertion is None or str(assertion[0]) != "accepted": + conn.rollback() + raise RuntimeError("frontier judgment assertion must be explicitly accepted before replanning") + if judgment_disposition != "retain_canonical_authority": + conn.rollback() + raise RuntimeError("frontier judgment resolution requires disposition=retain_canonical_authority") + observed = stored_plan + else: + input_raw_ids = tuple(str(value) for value in json.loads(str(row["input_raw_ids_json"]))) + observed = build_raw_replay_plan(conn, input_raw_ids) now = int(time.time() * 1000) full_receipt = json_document( { @@ -1342,6 +1433,8 @@ def resolve_raw_authority_blocker(archive_root: Path, blocker_id: str, *, resolu "superseded_plan_id": str(row["plan_id"]), "current_plan": observed.to_dict(), "operator_resolution": resolution.strip(), + "operator_assertion_id": assertion_id, + "judgment_disposition": judgment_disposition, "resolved_at_ms": now, } ) @@ -1369,6 +1462,8 @@ def resolve_raw_authority_blocker(archive_root: Path, blocker_id: str, *, resolu "logical_key_count": len(observed.logical_keys), }, "operator_resolution": resolution.strip(), + "operator_assertion_id": assertion_id, + "judgment_disposition": judgment_disposition, "resolved_at_ms": now, "detail_query_handle": raw_authority_detail_query_handle(str(row["census_id"]), str(row["plan_id"])), } @@ -1532,6 +1627,7 @@ def reject_invalid_raw_replay_application( "reject_stale_raw_replay_plan", "resolve_raw_authority_blocker", "unresolved_raw_authority_blockers", + "unresolved_raw_replay_blockers", "validate_raw_replay_plan", "validate_raw_replay_application_receipt", ] diff --git a/polylogue/storage/raw_reconciler.py b/polylogue/storage/raw_reconciler.py new file mode 100644 index 0000000000..f31bec5e20 --- /dev/null +++ b/polylogue/storage/raw_reconciler.py @@ -0,0 +1,1422 @@ +"""Proof-driven census for every accepted raw-authority frontier. + +This module owns the provider-neutral state machine. Historical incident +actuators remain implementation strategies in :mod:`polylogue.storage.repair`; +they do not get to define separate public notions of plan identity, evidence, +or readiness. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import json +import sqlite3 +import time +from collections import Counter +from collections.abc import Iterator, Sequence +from contextlib import closing +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from typing import TYPE_CHECKING, cast + +from polylogue.config import Config +from polylogue.core.json import JSONDocument, json_document +from polylogue.logging import get_logger +from polylogue.paths import archive_file_set_root_for_paths +from polylogue.storage.blob_store import BlobStore +from polylogue.storage.raw_authority import ( + RawAuthorityCensusReceipt, + RawReplayPlan, + RawReplayPlanOutcome, + RawReplayPlanStatus, + finalize_raw_authority_census, + raw_authority_detail_query_handle, + record_raw_authority_census, + record_raw_replay_outcome, +) +from polylogue.storage.sqlite.archive_tiers.source_write import deterministic_raw_session_id + +logger = get_logger(__name__) + +if TYPE_CHECKING: + from polylogue.storage.repair import ( + BrowserCaptureOriginRepairItem, + DuplicateRawIdentityRepairItem, + QuarantinedAcceptedRawRepairItem, + ) + + +class RawAuthorityFrontierState(StrEnum): + """Mutually exclusive authority states for one accepted frontier.""" + + PROVEN_CURRENT = "proven_current" + SAFELY_REKEYABLE = "safely_rekeyable" + DUPLICATE_ALIAS = "duplicate_alias" + SUPERSEDED = "superseded" + MISSING_BYTES_REACQUIRE = "missing_bytes_reacquire" + CONFLICTING_AUTHORITY_NEEDS_JUDGMENT = "conflicting_authority_needs_judgment" + UNRESOLVED_PROVENANCE = "unresolved_provenance" + CORRUPT = "corrupt" + + +class RawAuthorityActuator(StrEnum): + """Strategies admitted behind the shared plan/apply/postflight contract.""" + + NONE = "none" + REPLAY = "raw_revision_replay" + REFINE_QUARANTINE = "refine_quarantined_raw" + COPY_FORWARD_ORIGIN = "copy_forward_origin" + FOLD_DUPLICATE_ALIAS = "fold_duplicate_alias" + REACQUIRE = "reacquire" + REQUEST_JUDGMENT = "request_judgment" + RESOLVE_CONFLICT = "resolve_conflict" + + +_EXECUTABLE_STATES = { + RawAuthorityFrontierState.SAFELY_REKEYABLE, + RawAuthorityFrontierState.DUPLICATE_ALIAS, +} + +_VERIFIED_BLOB_STATS: dict[str, tuple[int, int, int, int, int]] = {} + + +def _canonical_json(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def _digest(value: object) -> str: + return hashlib.sha256(_canonical_json(value).encode()).hexdigest() + + +def _json_value(value: object) -> object: + if isinstance(value, (bytes, memoryview)): + return bytes(value).hex() + return value + + +@dataclass(frozen=True, slots=True) +class RawAuthorityFrontierItem: + """One complete, stable, evidence-bound frontier classification.""" + + state: RawAuthorityFrontierState + actuator: RawAuthorityActuator + raw_id: str + logical_source_key: str | None + session_id: str | None + reason: str + evidence_digest: str + input_raw_ids: tuple[str, ...] + source_preconditions: JSONDocument + index_preconditions: JSONDocument + strategy_witness: JSONDocument + plan_id: str + evidence_ref: str | None = None + + def to_dict(self) -> JSONDocument: + return json_document(dataclasses.asdict(self)) + + @property + def executable(self) -> bool: + return self.state in _EXECUTABLE_STATES + + +@dataclass(frozen=True, slots=True) +class _StrategyOverride: + state: RawAuthorityFrontierState + actuator: RawAuthorityActuator + reason: str + witness: JSONDocument + input_raw_ids: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class RawAuthorityFrontierCensus: + """One persisted census over accepted heads plus terminal supersessions.""" + + census_id: str + query_handle: str + inventory_digest: str + plan_inventory_digest: str + state_counts: JSONDocument + accepted_head_count: int + terminal_superseded_count: int + plan_count: int + executable_plan_count: int + items: tuple[RawAuthorityFrontierItem, ...] + + def to_dict(self, *, sample_limit: int = 100) -> JSONDocument: + sample = self.items[:sample_limit] + return json_document( + { + "schema": "polylogue.raw-authority-frontier-census.v1", + "census_id": self.census_id, + "query_handle": self.query_handle, + "inventory_digest": self.inventory_digest, + "plan_inventory_digest": self.plan_inventory_digest, + "state_counts": self.state_counts, + "accepted_head_count": self.accepted_head_count, + "terminal_superseded_count": self.terminal_superseded_count, + "plan_count": self.plan_count, + "executable_plan_count": self.executable_plan_count, + "returned_count": len(sample), + "items_truncated": len(sample) < len(self.items), + "items": [item.to_dict() for item in sample], + } + ) + + +@dataclass(frozen=True, slots=True) +class RawAuthorityFrontierApplyReport: + """Bounded receipt for one shared-contract apply pass.""" + + census_id: str + preview_census_id: str + selected_plan_count: int + executed_plan_count: int + retryable_plan_count: int + post_inventory_digest: str + post_plan_count: int + outcome_refs: tuple[str, ...] + + @property + def success(self) -> bool: + return self.retryable_plan_count == 0 + + def to_dict(self) -> JSONDocument: + return json_document(dataclasses.asdict(self) | {"success": self.success}) + + +def _archive_root(config: Config) -> Path: + return archive_file_set_root_for_paths(archive_root_path=config.archive_root, db_anchor=config.db_path) + + +def _rows(cursor: sqlite3.Cursor) -> list[dict[str, object]]: + names = tuple(column[0] for column in cursor.description or ()) + return [{name: _json_value(value) for name, value in zip(names, row, strict=True)} for row in cursor.fetchall()] + + +def _chunks(values: Sequence[str], size: int = 100) -> Iterator[list[str]]: + """Yield bounded strategy-proof requests in deterministic order.""" + for start in range(0, len(values), size): + yield list(values[start : start + size]) + + +def _verified_blob_bytes(blob_store: BlobStore, hash_hex: str) -> bool: + """Hash once per stable on-disk inode state, then reuse the process receipt.""" + path = blob_store.blob_path(hash_hex) + try: + stat = path.stat() + except OSError: + return False + fingerprint = (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns) + if _VERIFIED_BLOB_STATS.get(hash_hex) == fingerprint: + return True + if not blob_store.verify(hash_hex): + return False + _VERIFIED_BLOB_STATS[hash_hex] = fingerprint + return True + + +def _browser_strategy_witness(item: BrowserCaptureOriginRepairItem) -> JSONDocument: + from polylogue.storage.repair import _browser_origin_item_payload + + return json_document( + { + "schema": "polylogue.raw-authority-strategy-witness.v1", + "kind": "browser_origin", + "item": _browser_origin_item_payload(item), + } + ) + + +def _quarantine_strategy_witness(item: QuarantinedAcceptedRawRepairItem) -> JSONDocument: + payload = { + key: _json_value(value) + for key, value in dataclasses.asdict(item).items() + if key not in {"proof_digest", "reason", "repaired", "status"} + } + return json_document( + { + "schema": "polylogue.raw-authority-strategy-witness.v1", + "kind": "quarantine_refinement", + "item": payload, + } + ) + + +def _duplicate_strategy_witness(item: DuplicateRawIdentityRepairItem) -> JSONDocument: + from polylogue.storage.repair import _duplicate_raw_identity_proof_digest + + return json_document( + { + "schema": "polylogue.raw-authority-strategy-witness.v1", + "kind": "duplicate_alias", + "proof_digest": _duplicate_raw_identity_proof_digest(item), + "stale_raw_id": item.stale_raw_id, + "canonical_raw_id": item.canonical_raw_id, + "session_id": item.session_id, + "logical_source_key": item.logical_source_key, + "accepted_source_revision": item.accepted_source_revision, + "accepted_content_hash": item.accepted_content_hash, + "accepted_frontier_kind": item.accepted_frontier_kind, + "accepted_frontier": item.accepted_frontier, + "accepted_decided_at_ms": item.accepted_decided_at_ms, + } + ) + + +def _browser_strategy_raw_ids(item: BrowserCaptureOriginRepairItem) -> tuple[str, ...]: + return tuple( + sorted( + { + raw_id + for raw_id in ( + item.raw_id, + item.replacement_raw_id, + item.copy_forward_raw_id, + item.semantic_canonical_raw_id, + *item.semantic_historical_raw_ids, + ) + if raw_id is not None + } + ) + ) + + +def _frontier_rows(conn: sqlite3.Connection) -> list[dict[str, object]]: + return _rows( + conn.execute( + """ + SELECT h.logical_source_key, h.session_id, + COALESCE(s.raw_id, h.accepted_raw_id) AS accepted_raw_id, + h.accepted_raw_id AS head_accepted_raw_id, + h.accepted_source_revision, + COALESCE(hex(s.content_hash), hex(h.accepted_content_hash)) AS accepted_content_hash, + h.accepted_frontier_kind, h.accepted_frontier, + h.decided_at_ms AS head_decided_at_ms, + s.origin AS session_origin, s.raw_id AS session_raw_id, + hex(s.content_hash) AS session_content_hash, + s.message_count, + r.origin AS raw_origin, r.capture_mode, r.native_id, + r.source_path, r.source_index, hex(r.blob_hash) AS blob_hash, + r.blob_size, r.logical_source_key AS raw_logical_source_key, + r.revision_kind, r.source_revision, r.predecessor_raw_id, + r.baseline_raw_id, r.append_start_offset, r.append_end_offset, + r.acquisition_generation, r.revision_authority + FROM index_tier.raw_revision_heads AS h + LEFT JOIN index_tier.sessions AS s ON s.session_id = h.session_id + LEFT JOIN raw_sessions AS r ON r.raw_id = COALESCE(s.raw_id, h.accepted_raw_id) + ORDER BY h.logical_source_key + """ + ) + ) + + +def _duplicate_alias_siblings(conn: sqlite3.Connection, row: dict[str, object]) -> tuple[str, ...]: + if row.get("raw_origin") is None or row.get("blob_hash") is None or row.get("native_id") is None: + return () + blob_hash = bytes.fromhex(cast(str, row["blob_hash"])) + expected_accepted = deterministic_raw_session_id( + str(row["raw_origin"]), + str(row["source_path"]), + int(cast(int, row["source_index"])), + blob_hash, + native_id=str(row["native_id"]), + ) + if expected_accepted != row["accepted_raw_id"]: + return () + siblings = conn.execute( + """ + SELECT raw_id + FROM raw_sessions + WHERE origin = ? AND source_path = ? AND source_index = ? + AND blob_hash = ? AND native_id IS NULL AND raw_id != ? + AND NOT EXISTS ( + SELECT 1 FROM index_tier.raw_revision_heads AS h + WHERE h.accepted_raw_id = raw_sessions.raw_id + ) + AND NOT EXISTS ( + SELECT 1 FROM index_tier.sessions AS s + WHERE s.raw_id = raw_sessions.raw_id + ) + ORDER BY raw_id + """, + ( + row["raw_origin"], + row["source_path"], + row["source_index"], + blob_hash, + row["accepted_raw_id"], + ), + ).fetchall() + expected_canonical = deterministic_raw_session_id( + str(row["raw_origin"]), + str(row["source_path"]), + int(cast(int, row["source_index"])), + blob_hash, + native_id=None, + ) + return tuple(str(sibling[0]) for sibling in siblings if str(sibling[0]) == expected_canonical) + + +def _item( + *, + state: RawAuthorityFrontierState, + actuator: RawAuthorityActuator, + row: dict[str, object], + reason: str, + input_raw_ids: tuple[str, ...] | None = None, + strategy_witness: JSONDocument | None = None, +) -> RawAuthorityFrontierItem: + raw_id = str(row["accepted_raw_id"]) + source = json_document( + { + key: row.get(key) + for key in ( + "raw_origin", + "capture_mode", + "native_id", + "source_path", + "source_index", + "blob_hash", + "blob_size", + "raw_logical_source_key", + "revision_kind", + "source_revision", + "predecessor_raw_id", + "baseline_raw_id", + "append_start_offset", + "append_end_offset", + "acquisition_generation", + "revision_authority", + ) + } + ) + index = json_document( + { + key: row.get(key) + for key in ( + "logical_source_key", + "session_id", + "accepted_raw_id", + "head_accepted_raw_id", + "accepted_source_revision", + "accepted_content_hash", + "accepted_frontier_kind", + "accepted_frontier", + "head_decided_at_ms", + "session_origin", + "session_raw_id", + "session_content_hash", + "message_count", + ) + } + ) + ids = tuple(sorted(set(input_raw_ids or (raw_id,)))) + evidence = { + "schema": "polylogue.raw-authority-frontier-evidence.v1", + "state": state.value, + "actuator": actuator.value, + "input_raw_ids": ids, + "source": source, + "index": index, + "strategy_witness": strategy_witness or {}, + } + evidence_digest = _digest(evidence) + plan_id = f"raw-authority-frontier:{evidence_digest}" + return RawAuthorityFrontierItem( + state=state, + actuator=actuator, + raw_id=raw_id, + logical_source_key=(str(row["logical_source_key"]) if row.get("logical_source_key") is not None else None), + session_id=(str(row["session_id"]) if row.get("session_id") is not None else None), + reason=reason, + evidence_digest=evidence_digest, + input_raw_ids=ids, + source_preconditions=source, + index_preconditions=index, + strategy_witness=strategy_witness or json_document({}), + plan_id=plan_id, + ) + + +def _classify_frontier( + conn: sqlite3.Connection, + blob_store: BlobStore, + row: dict[str, object], + strategy_override: _StrategyOverride | None, +) -> RawAuthorityFrontierItem: + raw_id = str(row["accepted_raw_id"]) + if row.get("raw_origin") is None: + return _item( + state=RawAuthorityFrontierState.MISSING_BYTES_REACQUIRE, + actuator=RawAuthorityActuator.REACQUIRE, + row=row, + reason="accepted head raw is absent from the durable source tier", + ) + blob_hash = str(row["blob_hash"]).lower() + blob_exists = blob_store.exists(blob_hash) + reacquisition_proven = blob_exists and _verified_blob_bytes(blob_store, blob_hash) + if not blob_exists or not reacquisition_proven: + return _item( + state=RawAuthorityFrontierState.MISSING_BYTES_REACQUIRE, + actuator=RawAuthorityActuator.REACQUIRE, + row=row, + reason="accepted head raw bytes do not prove the expected content-addressed digest", + ) + if row.get("session_id") is None or row.get("session_origin") is None: + return _item( + state=RawAuthorityFrontierState.CORRUPT, + actuator=RawAuthorityActuator.NONE, + row=row, + reason="accepted head has no matching materialized session", + ) + if row.get("session_raw_id") != raw_id or row.get("session_content_hash") != row.get("accepted_content_hash"): + return _item( + state=RawAuthorityFrontierState.CORRUPT, + actuator=RawAuthorityActuator.NONE, + row=row, + reason="accepted head and materialized session authority disagree", + ) + duplicate_siblings = _duplicate_alias_siblings(conn, row) + if duplicate_siblings and row.get("native_id") is not None: + from polylogue.storage.repair import _inspect_duplicate_raw_identity + + if len(duplicate_siblings) != 1: + raise RuntimeError(f"duplicate alias classification is not injective for {raw_id}") + with closing(sqlite3.connect(f"file:{blob_store.root.parent / 'index.db'}?mode=ro", uri=True)) as proof_conn: + proof_conn.row_factory = sqlite3.Row + proof_conn.execute( + "ATTACH DATABASE ? AS source", + (f"file:{blob_store.root.parent / 'source.db'}?mode=ro",), + ) + duplicate_item = _inspect_duplicate_raw_identity( + proof_conn, + blob_store.root.parent, + raw_id, + duplicate_siblings[0], + ) + if duplicate_item.status not in {"eligible", "already_repaired"}: + raise RuntimeError(f"duplicate alias lacks an exact strategy proof: {duplicate_item.reason}") + duplicate_witness = _duplicate_strategy_witness(duplicate_item) + return _item( + state=RawAuthorityFrontierState.DUPLICATE_ALIAS, + actuator=RawAuthorityActuator.FOLD_DUPLICATE_ALIAS, + row=row, + reason="accepted raw uses the obsolete native-id-inclusive identity while an exact canonical twin exists", + input_raw_ids=(raw_id, *duplicate_siblings), + strategy_witness=duplicate_witness, + ) + if strategy_override is not None: + return _item( + state=strategy_override.state, + actuator=strategy_override.actuator, + row=row, + reason=strategy_override.reason, + strategy_witness=strategy_override.witness, + input_raw_ids=strategy_override.input_raw_ids, + ) + if row.get("head_accepted_raw_id") != raw_id: + return _item( + state=RawAuthorityFrontierState.CORRUPT, + actuator=RawAuthorityActuator.NONE, + row=row, + reason="accepted revision head and materialized session select different raw authority", + ) + if row.get("session_origin") != row.get("raw_origin"): + return _item( + state=RawAuthorityFrontierState.UNRESOLVED_PROVENANCE, + actuator=RawAuthorityActuator.NONE, + row=row, + reason="origin mismatch lacks a strategy proof admitted by the shared reconciler", + ) + if row.get("revision_authority") == "quarantined": + return _item( + state=RawAuthorityFrontierState.UNRESOLVED_PROVENANCE, + actuator=RawAuthorityActuator.REFINE_QUARANTINE, + row=row, + reason="accepted raw authority remains quarantined pending exact refinement proof", + ) + if row.get("raw_logical_source_key") != row.get("logical_source_key"): + return _item( + state=RawAuthorityFrontierState.UNRESOLVED_PROVENANCE, + actuator=RawAuthorityActuator.REPLAY, + row=row, + reason="accepted raw and index head logical authority keys disagree", + ) + return _item( + state=RawAuthorityFrontierState.PROVEN_CURRENT, + actuator=RawAuthorityActuator.NONE, + row=row, + reason="accepted source bytes, identity, head, and materialized session agree", + ) + + +def _strategy_overrides( + config: Config, + rows: list[dict[str, object]], +) -> dict[str, _StrategyOverride]: + """Ask legacy incident inspectors for proofs, never for plan identity.""" + from polylogue.storage.repair import ( + inspect_browser_canonical_authority_conflicts, + inspect_browser_capture_origin_mismatches, + inspect_quarantined_accepted_raws, + ) + + overrides: dict[str, _StrategyOverride] = {} + browser_ids = sorted( + { + str(row["accepted_raw_id"]) + for row in rows + if row.get("raw_origin") is not None and row.get("session_origin") != row.get("raw_origin") + } + ) + for browser_chunk in _chunks(browser_ids): + browser_items = inspect_browser_capture_origin_mismatches(config, browser_chunk) + for browser_item in browser_items: + if browser_item.status in {"eligible", "already_repaired"}: + overrides[browser_item.raw_id] = _StrategyOverride( + state=RawAuthorityFrontierState.SAFELY_REKEYABLE, + actuator=RawAuthorityActuator.COPY_FORWARD_ORIGIN, + reason="browser-origin strategy proved an exact evidence-preserving copy-forward", + witness=_browser_strategy_witness(browser_item), + input_raw_ids=_browser_strategy_raw_ids(browser_item), + ) + conflicts = inspect_browser_canonical_authority_conflicts(config, browser_chunk) + for conflict_item in conflicts.items: + if conflict_item.raw_id in overrides: + continue + if conflict_item.competing_raw_id is None: + overrides[conflict_item.raw_id] = _StrategyOverride( + state=RawAuthorityFrontierState.UNRESOLVED_PROVENANCE, + actuator=RawAuthorityActuator.NONE, + reason=( + "browser-origin evidence has a retained membership precondition but no " + "canonical authority that an operator could retain" + ), + witness=json_document( + { + "schema": "polylogue.raw-authority-strategy-witness.v1", + "kind": "browser_membership_precondition", + "evidence": dataclasses.asdict(conflict_item), + } + ), + input_raw_ids=(conflict_item.raw_id,), + ) + continue + overrides[conflict_item.raw_id] = _StrategyOverride( + state=RawAuthorityFrontierState.CONFLICTING_AUTHORITY_NEEDS_JUDGMENT, + actuator=RawAuthorityActuator.REQUEST_JUDGMENT, + reason=conflict_item.reason, + witness=json_document( + { + "schema": "polylogue.raw-authority-strategy-witness.v1", + "kind": "browser_conflict", + "evidence": dataclasses.asdict(conflict_item), + } + ), + input_raw_ids=tuple(sorted({conflict_item.raw_id, conflict_item.competing_raw_id})), + ) + quarantine_ids = sorted( + { + str(row["accepted_raw_id"]) + for row in rows + if row.get("revision_authority") == "quarantined" and str(row["accepted_raw_id"]) not in browser_ids + } + ) + for quarantine_chunk in _chunks(quarantine_ids): + quarantine_items = inspect_quarantined_accepted_raws(config, quarantine_chunk) + for quarantine_item in quarantine_items: + if quarantine_item.status in {"eligible", "already_repaired"}: + overrides[quarantine_item.raw_id] = _StrategyOverride( + state=RawAuthorityFrontierState.SAFELY_REKEYABLE, + actuator=RawAuthorityActuator.REFINE_QUARANTINE, + reason="quarantined-raw strategy proved exact accepted-byte and semantic authority", + witness=_quarantine_strategy_witness(quarantine_item), + input_raw_ids=tuple(sorted({quarantine_item.raw_id, *quarantine_item.census_stage_raw_ids})), + ) + return overrides + + +_OBLIGATION_STATES = { + RawAuthorityFrontierState.MISSING_BYTES_REACQUIRE, + RawAuthorityFrontierState.CONFLICTING_AUTHORITY_NEEDS_JUDGMENT, + RawAuthorityFrontierState.UNRESOLVED_PROVENANCE, + RawAuthorityFrontierState.CORRUPT, +} + + +def _record_judgment_candidate(config: Config, item: RawAuthorityFrontierItem, *, now_ms: int) -> tuple[str, bool]: + """Persist the conflict as a non-authoritative candidate for operator judgment.""" + from polylogue.core.enums import AssertionKind, AssertionStatus, AssertionVisibility + from polylogue.storage.sqlite.archive_tiers.user_write import read_assertion_envelope, upsert_assertion + + assertion_id = f"judgment:{_digest(['raw-authority-frontier', item.plan_id])}" + root = _archive_root(config) + with closing(sqlite3.connect(root / "user.db")) as conn, conn: + existing = read_assertion_envelope(conn, assertion_id) + if existing is not None and existing.status is not AssertionStatus.CANDIDATE: + return existing.assertion_id, False + upsert_assertion( + conn, + assertion_id=assertion_id, + scope_ref="insight:raw-authority-frontier@v1", + target_ref=f"session:{item.session_id}" if item.session_id is not None else f"raw:{item.raw_id}", + key=item.plan_id, + kind=AssertionKind.JUDGMENT, + value={ + "schema": "polylogue.raw-authority-judgment-request.v1", + "plan_id": item.plan_id, + "state": item.state.value, + "actuator": item.actuator.value, + "raw_id": item.raw_id, + "logical_source_key": item.logical_source_key, + "evidence_digest": item.evidence_digest, + "reason": item.reason, + "supported_dispositions": ["retain_canonical_authority"], + }, + body_text=item.reason, + author_ref="insight:raw-authority-frontier@v1", + author_kind="detector", + status=AssertionStatus.CANDIDATE, + visibility=AssertionVisibility.PRIVATE, + context_policy={"inject": False, "promotion_required": True}, + now_ms=now_ms, + ) + return assertion_id, False + + +def _apply_judgment_dispositions( + config: Config, + items: tuple[RawAuthorityFrontierItem, ...], +) -> tuple[RawAuthorityFrontierItem, ...]: + """Promote explicitly resolved conflict plans into executable successors.""" + root = _archive_root(config) + with closing(sqlite3.connect(f"file:{root / 'source.db'}?mode=ro", uri=True)) as conn: + resolutions = { + str(plan_id): json_document(json.loads(str(resolution))) + for plan_id, resolution in conn.execute( + """ + SELECT plan_id, resolution + FROM raw_authority_blockers + WHERE resolved_at_ms IS NOT NULL AND resolution IS NOT NULL + ORDER BY resolved_at_ms + """ + ) + } + promoted: list[RawAuthorityFrontierItem] = [] + for item in items: + resolution = resolutions.get(item.plan_id) + disposition = None if resolution is None else resolution.get("judgment_disposition") + if ( + item.state is not RawAuthorityFrontierState.CONFLICTING_AUTHORITY_NEEDS_JUDGMENT + or disposition != "retain_canonical_authority" + ): + promoted.append(item) + continue + assert resolution is not None + witness = json_document( + { + "schema": "polylogue.raw-authority-strategy-witness.v1", + "kind": "browser_conflict_resolution", + "conflict": item.strategy_witness, + "judgment": { + "disposition": disposition, + "operator_assertion_id": resolution.get("operator_assertion_id"), + "superseded_plan_id": item.plan_id, + }, + } + ) + evidence = { + "schema": "polylogue.raw-authority-frontier-evidence.v1", + "state": RawAuthorityFrontierState.SAFELY_REKEYABLE.value, + "actuator": RawAuthorityActuator.RESOLVE_CONFLICT.value, + "input_raw_ids": item.input_raw_ids, + "source": item.source_preconditions, + "index": item.index_preconditions, + "strategy_witness": witness, + } + evidence_digest = _digest(evidence) + promoted.append( + dataclasses.replace( + item, + state=RawAuthorityFrontierState.SAFELY_REKEYABLE, + actuator=RawAuthorityActuator.RESOLVE_CONFLICT, + reason="accepted operator judgment retained the exact canonical authority", + evidence_digest=evidence_digest, + strategy_witness=witness, + plan_id=f"raw-authority-frontier:{evidence_digest}", + evidence_ref=None, + ) + ) + return tuple(promoted) + + +def _reconcile_frontier_obligations( + config: Config, + census_id: str, + items: tuple[RawAuthorityFrontierItem, ...], +) -> None: + """Publish current obligations and close only those disproven by a later census.""" + root = _archive_root(config) + now = int(time.time() * 1000) + blocking = tuple(item for item in items if item.state in _OBLIGATION_STATES) + judgment_results = { + item.plan_id: _record_judgment_candidate(config, item, now_ms=now) + for item in blocking + if item.state is RawAuthorityFrontierState.CONFLICTING_AUTHORITY_NEEDS_JUDGMENT + } + judgment_refs = {plan_id: result[0] for plan_id, result in judgment_results.items()} + judged_plan_ids = {plan_id for plan_id, result in judgment_results.items() if result[1]} + current_ids = {item.plan_id for item in blocking} - judged_plan_ids + with closing(sqlite3.connect(root / "source.db")) as conn, conn: + for item in blocking: + if item.plan_id in judged_plan_ids: + continue + blocker_id = f"raw-authority-blocker:{_digest(['frontier', census_id, item.plan_id])}" + observed = { + "schema": "polylogue.raw-authority-frontier-obligation.v1", + "state": item.state.value, + "actuator": item.actuator.value, + "reason": item.reason, + "evidence_digest": item.evidence_digest, + } + judgment_assertion_id = judgment_refs.get(item.plan_id) + if judgment_assertion_id is not None: + observed["judgment_assertion_id"] = judgment_assertion_id + conn.execute( + """ + INSERT INTO raw_authority_blockers ( + blocker_id, plan_id, census_id, reason, expected_json, + observed_json, created_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT DO NOTHING + """, + ( + blocker_id, + item.plan_id, + census_id, + item.reason, + _canonical_json(_plan(item).to_dict()), + _canonical_json(observed), + now, + ), + ) + open_rows = conn.execute( + """ + SELECT b.blocker_id, b.plan_id + FROM raw_authority_blockers AS b + JOIN raw_authority_plans AS p ON p.plan_id = b.plan_id + WHERE b.resolved_at_ms IS NULL + AND json_extract(p.authority_witness_json, '$.schema') = + 'polylogue.raw-authority-frontier-plan.v1' + """ + ).fetchall() + for blocker_id, plan_id in open_rows: + plan_id_text = str(plan_id) + if plan_id_text in current_ids: + continue + conn.execute( + """ + UPDATE raw_authority_blockers + SET resolved_at_ms = ?, resolution = ? + WHERE blocker_id = ? AND resolved_at_ms IS NULL + """, + ( + now, + _canonical_json( + { + "schema": "polylogue.raw-authority-obligation-resolution.v1", + "reason": ( + "an accepted operator judgment acknowledged the retained conflict" + if plan_id_text in judged_plan_ids + else "a later complete frontier census disproved the prior blocking state" + ), + "successor_census_id": census_id, + } + ), + blocker_id, + ), + ) + + +def _terminal_superseded_items(conn: sqlite3.Connection) -> list[RawAuthorityFrontierItem]: + rows = _rows( + conn.execute( + """ + SELECT a.logical_source_key, a.session_id, a.raw_id AS accepted_raw_id, + a.source_revision AS accepted_source_revision, + hex(a.accepted_content_hash) AS accepted_content_hash, + NULL AS accepted_frontier_kind, NULL AS accepted_frontier, + a.decided_at_ms AS head_decided_at_ms, + s.origin AS session_origin, s.raw_id AS session_raw_id, + hex(s.content_hash) AS session_content_hash, s.message_count, + r.origin AS raw_origin, r.capture_mode, r.native_id, + r.source_path, r.source_index, hex(r.blob_hash) AS blob_hash, + r.blob_size, r.logical_source_key AS raw_logical_source_key, + r.revision_kind, r.source_revision, r.predecessor_raw_id, + r.baseline_raw_id, r.append_start_offset, r.append_end_offset, + r.acquisition_generation, r.revision_authority + FROM index_tier.raw_revision_applications AS a + JOIN raw_sessions AS r ON r.raw_id = a.raw_id + LEFT JOIN index_tier.sessions AS s ON s.session_id = a.session_id + LEFT JOIN index_tier.raw_revision_heads AS h ON h.accepted_raw_id = a.raw_id + WHERE a.decision = 'superseded' AND h.accepted_raw_id IS NULL + ORDER BY a.raw_id, a.logical_source_key + """ + ) + ) + return [ + _item( + state=RawAuthorityFrontierState.SUPERSEDED, + actuator=RawAuthorityActuator.NONE, + row=row, + reason="durable application receipt terminally supersedes this retained snapshot", + ) + for row in rows + ] + + +def _plan(item: RawAuthorityFrontierItem) -> RawReplayPlan: + witness = json_document( + { + "schema": "polylogue.raw-authority-frontier-plan.v1", + "state": item.state.value, + "actuator": item.actuator.value, + "reason": item.reason, + "evidence_digest": item.evidence_digest, + "strategy_witness": item.strategy_witness, + } + ) + return RawReplayPlan( + plan_id=item.plan_id, + input_digest=item.evidence_digest, + input_raw_ids=item.input_raw_ids, + logical_keys=((item.logical_source_key,) if item.logical_source_key is not None else ()), + authority_witness=witness, + source_preconditions=item.source_preconditions, + index_preconditions=item.index_preconditions, + ) + + +def _frontier_items(config: Config) -> tuple[tuple[RawAuthorityFrontierItem, ...], int, int]: + root = _archive_root(config) + source_db = root / "source.db" + index_db = root / "index.db" + if not source_db.is_file() or not index_db.is_file(): + raise RuntimeError("raw authority frontier census requires initialized source and index tiers") + with closing(sqlite3.connect(source_db)) as conn: + conn.row_factory = sqlite3.Row + conn.execute("ATTACH DATABASE ? AS index_tier", (str(index_db),)) + head_rows = _frontier_rows(conn) + overrides = _strategy_overrides(config, head_rows) + head_items = [ + _classify_frontier(conn, BlobStore(root / "blob"), row, overrides.get(str(row["accepted_raw_id"]))) + for row in head_rows + ] + superseded_items = _terminal_superseded_items(conn) + all_items = _apply_judgment_dispositions(config, (*head_items, *superseded_items)) + return ( + tuple(sorted(all_items, key=lambda item: (item.raw_id, item.plan_id))), + len(head_items), + len(superseded_items), + ) + + +def _state_counts(items: tuple[RawAuthorityFrontierItem, ...]) -> JSONDocument: + return json_document(dict(sorted(Counter(item.state.value for item in items).items()))) + + +def _residual(state_counts: JSONDocument) -> JSONDocument: + return json_document( + { + "schema": "polylogue.raw-authority-frontier-residual.v1", + "state_counts": { + state: count + for state, count in state_counts.items() + if state != RawAuthorityFrontierState.PROVEN_CURRENT.value + }, + } + ) + + +def inspect_raw_authority_frontier(config: Config) -> RawAuthorityFrontierCensus: + """Persist one complete accepted-frontier census without applying repairs. + + A census is not a read operation: publishing it also reconciles durable + frontier blockers and judgment obligations. Offline callers therefore + need the same daemon exclusion boundary as an apply; daemon convergence is + admitted through its active write lease. + """ + from polylogue.maintenance.offline_guard import offline_maintenance_block_reason + + block_reason = offline_maintenance_block_reason(config, active=True, dry_run=False) + if block_reason is not None: + raise RuntimeError(block_reason) + root = _archive_root(config) + all_items, accepted_head_count, terminal_superseded_count = _frontier_items(config) + state_counts_counter = Counter(item.state.value for item in all_items) + state_counts = json_document(dict(sorted(state_counts_counter.items()))) + inventory_digest = _digest([item.to_dict() for item in all_items]) + gap_items = tuple(item for item in all_items if item.state is not RawAuthorityFrontierState.PROVEN_CURRENT) + plans = tuple(_plan(item) for item in gap_items) + executable_ids = {item.plan_id for item in gap_items if item.executable} + receipt: RawAuthorityCensusReceipt = record_raw_authority_census( + root, + plans, + selected_plan_ids=set(), + executable_plan_ids=executable_ids, + mode="dry_run", + quiescent=True, + scope={ + "schema": "polylogue.raw-authority-frontier-scope.v1", + "accepted_head_count": accepted_head_count, + "terminal_superseded_count": terminal_superseded_count, + "inventory_digest": inventory_digest, + "state_counts": state_counts, + }, + residual=_residual(state_counts), + ) + _reconcile_frontier_obligations(config, receipt.census_id, all_items) + bound_items = tuple( + dataclasses.replace( + item, + evidence_ref=( + raw_authority_detail_query_handle(receipt.census_id, item.plan_id) + if item.state is not RawAuthorityFrontierState.PROVEN_CURRENT + else None + ), + ) + for item in all_items + ) + return RawAuthorityFrontierCensus( + census_id=receipt.census_id, + query_handle=receipt.query_handle, + inventory_digest=inventory_digest, + plan_inventory_digest=receipt.inventory_digest, + state_counts=state_counts, + accepted_head_count=accepted_head_count, + terminal_superseded_count=terminal_superseded_count, + plan_count=len(plans), + executable_plan_count=len(executable_ids), + items=bound_items, + ) + + +def _preview_plan_ids(root: Path, census_id: str) -> set[str]: + with closing(sqlite3.connect(f"file:{root / 'source.db'}?mode=ro", uri=True)) as conn: + row = conn.execute( + "SELECT mode, lifecycle_status FROM raw_authority_censuses WHERE census_id = ?", + (census_id,), + ).fetchone() + if row is None: + raise KeyError(census_id) + if tuple(row) != ("dry_run", "completed"): + raise RuntimeError("raw authority apply requires a completed dry-run frontier census") + return { + str(plan_id) + for (plan_id,) in conn.execute( + "SELECT plan_id FROM raw_authority_census_plans WHERE census_id = ?", + (census_id,), + ) + } + + +def _apply_strategy( + config: Config, + item: RawAuthorityFrontierItem, +) -> JSONDocument: + from polylogue.storage.index_generation import RebuildLease + from polylogue.storage.repair import ( + _apply_browser_conflict_canonical_resolution, + _apply_browser_origin_repair_item, + _apply_duplicate_raw_identity_repair, + _attach_repair_index, + _browser_origin_strategy_terminal, + _cas_refine_quarantined_accepted_raw, + _inspect_browser_capture_origin_strategy, + _inspect_duplicate_raw_identity, + _inspect_quarantined_accepted_raw, + _stage_browser_origin_copy_forward_source, + _stage_quarantined_census_cohort, + _validate_quarantined_raw_repair_blob_budget, + _verify_browser_origin_copy_forward_source_stage, + ) + + root = _archive_root(config) + source_db = root / "source.db" + index_db = root / "index.db" + + if item.actuator is RawAuthorityActuator.RESOLVE_CONFLICT: + conflict = item.strategy_witness.get("conflict") + judgment = item.strategy_witness.get("judgment") + if not isinstance(conflict, dict) or not isinstance(judgment, dict): + raise RuntimeError("conflict-resolution strategy witness is incomplete") + evidence = conflict.get("evidence") + if not isinstance(evidence, dict) or judgment.get("disposition") != "retain_canonical_authority": + raise RuntimeError("conflict-resolution strategy is not explicitly authorized") + with RebuildLease(root), closing(sqlite3.connect(f"file:{index_db}?mode=rw", uri=True)) as conn: + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + conn.execute("ATTACH DATABASE ? AS source", (f"file:{source_db}?mode=ro",)) + conn.execute("BEGIN IMMEDIATE") + try: + _apply_browser_conflict_canonical_resolution(root, conn, item.raw_id, evidence) + conn.commit() + except Exception: + conn.rollback() + raise + return json_document( + { + "strategy": item.actuator.value, + "disposition": judgment["disposition"], + "repaired_count": 1, + } + ) + if item.actuator is RawAuthorityActuator.FOLD_DUPLICATE_ALIAS: + canonical_ids = tuple(raw_id for raw_id in item.input_raw_ids if raw_id != item.raw_id) + if len(canonical_ids) != 1: + raise RuntimeError("duplicate-alias plan does not identify exactly one canonical twin") + with RebuildLease(root), closing(sqlite3.connect(f"file:{index_db}?mode=rw", uri=True)) as conn: + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + conn.execute("ATTACH DATABASE ? AS source", (f"file:{source_db}?mode=ro",)) + conn.execute("BEGIN IMMEDIATE") + try: + duplicate_locked = _inspect_duplicate_raw_identity(conn, root, item.raw_id, canonical_ids[0]) + if _duplicate_strategy_witness(duplicate_locked) != item.strategy_witness: + raise RuntimeError("duplicate strategy proof changed after plan authorization") + if duplicate_locked.status == "eligible": + _apply_duplicate_raw_identity_repair(conn, duplicate_locked) + elif duplicate_locked.status != "already_repaired": + raise RuntimeError(f"duplicate strategy lost its exact proof: {duplicate_locked.reason}") + after = _inspect_duplicate_raw_identity(conn, root, item.raw_id, canonical_ids[0]) + if after.status != "already_repaired": + raise RuntimeError("duplicate strategy did not reach its typed terminal postcondition") + conn.commit() + except Exception: + conn.rollback() + raise + return json_document( + { + "strategy": item.actuator.value, + "repaired_count": int(duplicate_locked.status == "eligible"), + "already_repaired_count": int(duplicate_locked.status == "already_repaired"), + } + ) + if item.actuator is RawAuthorityActuator.COPY_FORWARD_ORIGIN: + from polylogue.storage.blob_publication import exclude_archive_blob_publishers + + with RebuildLease(root), exclude_archive_blob_publishers(source_db): + with closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True)) as proof_conn: + proof_conn.execute("ATTACH DATABASE ? AS source", (str(source_db),)) + preview = _inspect_browser_capture_origin_strategy(root, item.raw_id, conn=proof_conn) + if _browser_strategy_witness(preview) != item.strategy_witness: + raise RuntimeError("browser-origin strategy proof changed after plan authorization") + if preview.status == "eligible" and preview.repair_strategy == "copy_forward": + with closing(sqlite3.connect(f"file:{source_db}?mode=rw", uri=True)) as source_conn: + source_conn.execute("PRAGMA foreign_keys = ON") + source_conn.execute("BEGIN IMMEDIATE") + try: + if not preview.copy_forward_source_complete: + _verify_browser_origin_copy_forward_source_stage(root, source_conn, preview) + _stage_browser_origin_copy_forward_source(source_conn, preview) + source_conn.commit() + except Exception: + source_conn.rollback() + raise + elif preview.status == "eligible" and preview.repair_strategy == "restore_canonical_head": + pass + elif preview.status != "already_repaired": + raise RuntimeError(f"browser-origin strategy lost its exact proof: {preview.reason}") + with closing(sqlite3.connect(f"file:{index_db}?mode=rw", uri=True)) as conn: + conn.execute("PRAGMA foreign_keys = ON") + conn.execute("ATTACH DATABASE ? AS source", (str(source_db),)) + conn.execute("BEGIN IMMEDIATE") + try: + browser_locked = _inspect_browser_capture_origin_strategy(root, item.raw_id, conn=conn) + if _browser_strategy_witness(browser_locked) != item.strategy_witness: + raise RuntimeError("browser-origin strategy proof changed under the apply transaction") + if browser_locked.status == "eligible": + _apply_browser_origin_repair_item(conn, browser_locked) + elif browser_locked.status != "already_repaired": + raise RuntimeError(f"browser-origin strategy lost its locked proof: {browser_locked.reason}") + if not _browser_origin_strategy_terminal(conn, browser_locked): + raise RuntimeError("browser-origin strategy did not reach its typed terminal postcondition") + conn.commit() + except Exception: + conn.rollback() + raise + return json_document( + { + "strategy": item.actuator.value, + "repaired_count": int(preview.status == "eligible"), + "already_repaired_count": int(preview.status == "already_repaired"), + } + ) + if item.actuator is RawAuthorityActuator.REFINE_QUARANTINE: + with RebuildLease(root), closing(sqlite3.connect(f"file:{source_db}?mode=rw", uri=True)) as source_conn: + source_conn.execute("PRAGMA foreign_keys = ON") + _attach_repair_index(source_conn, index_db) + source_conn.execute("BEGIN IMMEDIATE") + try: + _validate_quarantined_raw_repair_blob_budget(source_conn, [item.raw_id]) + quarantine_locked = _inspect_quarantined_accepted_raw(root, item.raw_id, conn=source_conn) + if _quarantine_strategy_witness(quarantine_locked) != item.strategy_witness: + raise RuntimeError("quarantine strategy proof changed after plan authorization") + if quarantine_locked.status == "eligible" and quarantine_locked.census_stage_raw_ids: + _stage_quarantined_census_cohort(source_conn, quarantine_locked) + quarantine_locked = _inspect_quarantined_accepted_raw(root, item.raw_id, conn=source_conn) + if quarantine_locked.status == "eligible": + _cas_refine_quarantined_accepted_raw(source_conn, quarantine_locked) + elif quarantine_locked.status != "already_repaired": + raise RuntimeError(f"quarantine strategy lost its exact proof: {quarantine_locked.reason}") + quarantine_after = _inspect_quarantined_accepted_raw(root, item.raw_id, conn=source_conn) + if quarantine_after.status != "already_repaired": + raise RuntimeError("quarantine strategy did not reach its typed terminal postcondition") + source_conn.commit() + except Exception: + source_conn.rollback() + raise + return json_document( + { + "strategy": item.actuator.value, + "repaired_count": int(quarantine_locked.status == "eligible"), + "already_repaired_count": int(quarantine_locked.status == "already_repaired"), + } + ) + raise RuntimeError(f"raw authority plan actuator is not automatically executable: {item.actuator.value}") + + +def apply_raw_authority_frontier( + config: Config, + *, + preview_census_id: str, + selected_plan_ids: tuple[str, ...], +) -> RawAuthorityFrontierApplyReport: + """Authorize, apply, receipt, and postflight exact plans through one contract.""" + from polylogue.maintenance.offline_guard import offline_maintenance_block_reason + + block_reason = offline_maintenance_block_reason(config, active=True, dry_run=False) + if block_reason is not None: + raise RuntimeError(block_reason) + if not selected_plan_ids or len(set(selected_plan_ids)) != len(selected_plan_ids): + raise ValueError("raw authority apply requires unique selected plan ids") + root = _archive_root(config) + preview_ids = _preview_plan_ids(root, preview_census_id) + unknown_preview_ids = set(selected_plan_ids) - preview_ids + if unknown_preview_ids: + raise RuntimeError(f"selected plans are absent from the preview census: {sorted(unknown_preview_ids)}") + before_items, accepted_head_count, terminal_superseded_count = _frontier_items(config) + current_by_plan = {item.plan_id: item for item in before_items} + missing_current_ids = set(selected_plan_ids) - set(current_by_plan) + if missing_current_ids: + raise RuntimeError(f"selected raw authority plans changed after preview: {sorted(missing_current_ids)}") + selected_items = tuple(current_by_plan[plan_id] for plan_id in selected_plan_ids) + if any(not item.executable for item in selected_items): + raise RuntimeError("raw authority apply selected a non-executable judgment/reacquisition/debt plan") + gap_items = tuple(item for item in before_items if item.state is not RawAuthorityFrontierState.PROVEN_CURRENT) + plans = tuple(_plan(item) for item in gap_items) + state_counts = _state_counts(before_items) + apply_receipt = record_raw_authority_census( + root, + plans, + selected_plan_ids=set(selected_plan_ids), + executable_plan_ids={item.plan_id for item in gap_items if item.executable}, + mode="apply", + quiescent=True, + scope={ + "schema": "polylogue.raw-authority-frontier-scope.v1", + "preview_census_id": preview_census_id, + "accepted_head_count": accepted_head_count, + "terminal_superseded_count": terminal_superseded_count, + "inventory_digest": _digest([item.to_dict() for item in before_items]), + "state_counts": state_counts, + }, + residual=_residual(state_counts), + ) + executed = 0 + retryable = 0 + outcome_refs: list[str] = [] + for item in selected_items: + try: + strategy_receipt = _apply_strategy(config, item) + outcome = RawReplayPlanOutcome( + plan_id=item.plan_id, + input_raw_ids=item.input_raw_ids, + status=RawReplayPlanStatus.EXECUTED, + reason="shared raw-authority plan reached its strategy terminal state", + next_action="none", + application_receipt=json_document( + { + "schema": "polylogue.raw-authority-frontier-application.v1", + "preview_census_id": preview_census_id, + "frontier_evidence_digest": item.evidence_digest, + "strategy": strategy_receipt, + } + ), + ) + executed += 1 + except Exception as exc: + logger.warning( + "raw authority strategy failed plan=%s actuator=%s", + item.plan_id, + item.actuator.value, + exc_info=True, + ) + outcome = RawReplayPlanOutcome( + plan_id=item.plan_id, + input_raw_ids=item.input_raw_ids, + status=RawReplayPlanStatus.RETRYABLE, + reason=f"strategy did not reach a proven terminal state: {type(exc).__name__}: {exc}", + next_action="reinspect the same immutable plan and retry after correcting the causal condition", + application_receipt=json_document( + { + "schema": "polylogue.raw-authority-frontier-application.v1", + "preview_census_id": preview_census_id, + "frontier_evidence_digest": item.evidence_digest, + } + ), + ) + retryable += 1 + record_raw_replay_outcome(root, apply_receipt.census_id, outcome) + outcome_refs.append(raw_authority_detail_query_handle(apply_receipt.census_id, item.plan_id)) + after_items, _after_head_count, _after_superseded_count = _frontier_items(config) + after_state_counts = _state_counts(after_items) + after_plans = tuple( + _plan(item) for item in after_items if item.state is not RawAuthorityFrontierState.PROVEN_CURRENT + ) + finalized = finalize_raw_authority_census( + root, + apply_receipt.census_id, + post_plans=after_plans, + post_residual=_residual(after_state_counts), + interrupted=retryable > 0, + ) + assert finalized.post_inventory_digest is not None + assert finalized.post_plan_count is not None + return RawAuthorityFrontierApplyReport( + census_id=apply_receipt.census_id, + preview_census_id=preview_census_id, + selected_plan_count=len(selected_items), + executed_plan_count=executed, + retryable_plan_count=retryable, + post_inventory_digest=finalized.post_inventory_digest, + post_plan_count=finalized.post_plan_count, + outcome_refs=tuple(outcome_refs), + ) + + +def recover_interrupted_raw_authority_frontier(config: Config) -> tuple[str, ...]: + """Conserve crash-left frontier applications from current typed evidence.""" + root = _archive_root(config) + source_db = root / "source.db" + if not source_db.is_file(): + return () + with closing(sqlite3.connect(source_db)) as conn: + conn.row_factory = sqlite3.Row + census_ids = [ + str(row[0]) + for row in conn.execute( + """ + SELECT census_id + FROM raw_authority_censuses + WHERE lifecycle_status = 'planned' + AND json_extract(scope_json, '$.schema') = + 'polylogue.raw-authority-frontier-scope.v1' + ORDER BY sequence_no + """ + ) + ] + if not census_ids: + return () + rows = conn.execute( + """ + SELECT c.census_id, p.* + FROM raw_authority_censuses AS c + JOIN raw_authority_census_plans AS cp ON cp.census_id = c.census_id + JOIN raw_authority_plans AS p ON p.plan_id = cp.plan_id + WHERE c.lifecycle_status = 'planned' + AND cp.selected = 1 AND cp.outcome_recorded = 0 + AND json_extract(p.authority_witness_json, '$.schema') = + 'polylogue.raw-authority-frontier-plan.v1' + ORDER BY c.sequence_no, cp.ordinal + """ + ).fetchall() + current_items, _head_count, _superseded_count = _frontier_items(config) + current_by_plan = {item.plan_id: item for item in current_items} + recovered: list[str] = [] + for row in rows: + census_id = str(row["census_id"]) + plan = RawReplayPlan( + plan_id=str(row["plan_id"]), + input_digest=str(row["input_digest"]), + input_raw_ids=tuple(str(value) for value in json.loads(str(row["input_raw_ids_json"]))), + logical_keys=tuple(str(value) for value in json.loads(str(row["logical_keys_json"]))), + authority_witness=json_document(json.loads(str(row["authority_witness_json"]))), + source_preconditions=json_document(json.loads(str(row["source_preconditions_json"]))), + index_preconditions=json_document(json.loads(str(row["index_preconditions_json"]))), + ) + current = current_by_plan.get(plan.plan_id) + related = tuple(item for item in current_items if set(item.input_raw_ids).intersection(plan.input_raw_ids)) + receipt = json_document( + { + "schema": "polylogue.raw-authority-frontier-recovery.v1", + "current_items": [item.to_dict() for item in related], + } + ) + if current is not None: + outcome = RawReplayPlanOutcome( + plan.plan_id, + plan.input_raw_ids, + RawReplayPlanStatus.RETRYABLE, + "interrupted before the immutable frontier plan reached terminal postconditions", + "retry the same immutable plan", + receipt, + ) + record_raw_replay_outcome(root, census_id, outcome) + elif related and all( + item.state in {RawAuthorityFrontierState.PROVEN_CURRENT, RawAuthorityFrontierState.SUPERSEDED} + for item in related + ): + outcome = RawReplayPlanOutcome( + plan.plan_id, + plan.input_raw_ids, + RawReplayPlanStatus.EXECUTED, + "interrupted application recovered from typed terminal frontier states", + "none", + receipt, + ) + record_raw_replay_outcome(root, census_id, outcome) + else: + from polylogue.storage.raw_authority import reject_stale_raw_replay_plan + + reject_stale_raw_replay_plan(root, census_id, plan, receipt) + recovered.append(plan.plan_id) + post_state_counts = _state_counts(current_items) + post_plans = tuple( + _plan(item) for item in current_items if item.state is not RawAuthorityFrontierState.PROVEN_CURRENT + ) + for census_id in census_ids: + finalize_raw_authority_census( + root, + census_id, + post_plans=post_plans, + post_residual=_residual(post_state_counts), + interrupted=True, + ) + return tuple(recovered) + + +__all__ = [ + "RawAuthorityActuator", + "RawAuthorityFrontierCensus", + "RawAuthorityFrontierApplyReport", + "RawAuthorityFrontierItem", + "RawAuthorityFrontierState", + "apply_raw_authority_frontier", + "inspect_raw_authority_frontier", + "recover_interrupted_raw_authority_frontier", +] diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 938554a19f..3a2eac4294 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -3,15 +3,13 @@ from __future__ import annotations import dataclasses -import fcntl import hashlib import json -import os import re import sqlite3 import time -from collections.abc import Callable, Iterator, Mapping, Sequence -from contextlib import closing, contextmanager, suppress +from collections.abc import Callable, Mapping, Sequence +from contextlib import closing from dataclasses import dataclass, field from datetime import UTC, datetime from pathlib import Path @@ -70,7 +68,6 @@ recover_interrupted_raw_authority_censuses, reject_invalid_raw_replay_application, reject_stale_raw_replay_plan, - unresolved_raw_authority_blockers, validate_raw_replay_application_receipt, validate_raw_replay_plan, ) @@ -87,11 +84,6 @@ _QUARANTINED_ACCEPTED_RAW_REPAIR_LIMIT = 100 _QUARANTINED_ACCEPTED_RAW_REPAIR_BLOB_LIMIT_BYTES = 256 * 1024 * 1024 _QUARANTINED_ACCEPTED_RAW_REPAIR_TOTAL_BLOB_LIMIT_BYTES = 512 * 1024 * 1024 -_QUARANTINED_ACCEPTED_RAW_REPAIR_RECEIPT_SCHEMA = "polylogue.quarantined-accepted-raw-repair.v1" -_BROWSER_CAPTURE_ORIGIN_REPAIR_RECEIPT_SCHEMA = "polylogue.browser-capture-origin-copy-forward.v1" -_LEGACY_BROWSER_CAPTURE_NATIVE_ID_REPAIR_RECEIPT_SCHEMA = "polylogue.browser-capture-legacy-native-id-copy-forward.v1" -_BYTE_PROVEN_BROWSER_CAPTURE_REKEY_RECEIPT_SCHEMA = "polylogue.browser-capture-byte-proven-rekey.v1" -_LEGACY_BROWSER_NATIVE_ID_TRANSACTION_PROTOCOL = "rollback-superjournal-v1" _QUARANTINED_CENSUS_STAGE_FINGERPRINT = "repair-quarantined-accepted-raw-v1" _QUARANTINED_CENSUS_STAGE_DETAIL = "census-only evidence staged before accepted-head authority refinement" _BROWSER_ORIGIN_SEMANTIC_HISTORICAL_WITNESS_LIMIT = 8 @@ -171,19 +163,6 @@ class QuarantinedAcceptedRawRepairItem: repaired: bool = False -@dataclass(frozen=True, slots=True) -class QuarantinedAcceptedRawRepairReport: - mode: str - requested_count: int - eligible_count: int - repaired_count: int - already_repaired_count: int - ineligible_count: int - proof_digest: str - receipt_path: str | None - items: tuple[QuarantinedAcceptedRawRepairItem, ...] - - @dataclass(frozen=True, slots=True) class BrowserCaptureOriginRepairItem: raw_id: str @@ -205,6 +184,7 @@ class BrowserCaptureOriginRepairItem: repair_strategy: str | None = None replacement_raw_id: str | None = None replacement_source_revision: str | None = None + replacement_content_hash: str | None = None replacement_frontier_kind: str | None = None replacement_frontier: int | None = None copy_forward_raw_id: str | None = None @@ -223,19 +203,6 @@ class BrowserCaptureOriginRepairItem: repaired: bool = False -@dataclass(frozen=True, slots=True) -class BrowserCaptureOriginRepairReport: - mode: str - requested_count: int - eligible_count: int - repaired_count: int - already_repaired_count: int - ineligible_count: int - proof_digest: str - receipt_path: str | None - items: tuple[BrowserCaptureOriginRepairItem, ...] - - @dataclass(frozen=True, slots=True) class _SemanticCanonicalWitness: raw_id: str @@ -264,10 +231,17 @@ class BrowserCanonicalAuthorityConflictWitness: old_logical_source_key: str | None = None canonical_logical_source_key: str | None = None unknown_raw_content_hash: str | None = None + unknown_source_revision: str | None = None + unknown_frontier_kind: str | None = None + unknown_frontier: int | None = None + unknown_decided_at_ms: int | None = None unknown_raw_message_count: int | None = None competing_raw_id: str | None = None competing_content_hash: str | None = None + competing_source_revision: str | None = None competing_frontier_kind: str | None = None + competing_frontier: int | None = None + competing_decided_at_ms: int | None = None competing_decision: str | None = None competing_message_count: int | None = None divergent_message_index: int | None = None @@ -1097,260 +1071,6 @@ def _inspect_quarantined_accepted_raw( return dataclasses.replace(item, proof_digest=_proof_digest(item)) -def _repair_receipt_targets(items: list[QuarantinedAcceptedRawRepairItem]) -> list[dict[str, object]]: - targets = [ - {key: value for key, value in dataclasses.asdict(item).items() if key not in {"reason", "repaired", "status"}} - for item in items - ] - return cast(list[dict[str, object]], json.loads(json.dumps(targets, sort_keys=True))) - - -def _repair_proof_digest(items: list[QuarantinedAcceptedRawRepairItem]) -> str: - proof_digests = [item.proof_digest for item in items] - return hashlib.sha256(json.dumps(proof_digests, separators=(",", ":")).encode()).hexdigest() - - -def _fsync_parent(path: Path) -> None: - descriptor = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) - try: - os.fsync(descriptor) - finally: - os.close(descriptor) - - -@dataclass(slots=True) -class _LockedQuarantinedRawRepairReceipt: - path: Path - descriptor: int - target_hash: str - terminal: bool - repair_intent_raw_ids: tuple[str, ...] - torn_terminals: tuple[bytes, ...] = () - receipt_terminated: bool = True - - def close(self) -> None: - fcntl.flock(self.descriptor, fcntl.LOCK_UN) - os.close(self.descriptor) - - -def _receipt_write(descriptor: int, payload: bytes) -> int: - return os.write(descriptor, payload) - - -def _write_receipt_all(descriptor: int, payload: bytes) -> None: - offset = 0 - while offset < len(payload): - written = _receipt_write(descriptor, payload[offset:]) - if written <= 0: - raise RuntimeError("operator repair receipt write made no progress") - offset += written - - -def _receipt_records(descriptor: int) -> tuple[list[dict[str, object] | bytes], bool]: - size = os.fstat(descriptor).st_size - if size > 16 * 1024 * 1024: - raise RuntimeError("existing repair receipt exceeds the bounded parser limit") - os.lseek(descriptor, 0, os.SEEK_SET) - chunks: list[bytes] = [] - remaining = size - while remaining: - chunk = os.read(descriptor, remaining) - if not chunk: - raise RuntimeError("existing repair receipt changed during its locked read") - chunks.append(chunk) - remaining -= len(chunk) - payload = b"".join(chunks) - terminated = payload.endswith(b"\n") - lines = payload.split(b"\n") - if terminated: - lines.pop() - records: list[dict[str, object] | bytes] = [] - for index, line in enumerate(lines): - if index == len(lines) - 1 and not terminated: - records.append(line) - continue - try: - parsed: object = json.loads(line.decode("utf-8")) - except (UnicodeDecodeError, ValueError, json.JSONDecodeError): - records.append(line) - continue - records.append(cast(dict[str, object], parsed) if isinstance(parsed, dict) else line) - return records, terminated - - -def _validate_repair_receipt_records( - parsed_receipt: tuple[list[dict[str, object] | bytes], bool], - *, - targets: list[dict[str, object]], - target_hash: str, -) -> tuple[bool, tuple[str, ...], tuple[bytes, ...], bool]: - records, terminated = parsed_receipt - if not records: - raise RuntimeError("existing repair receipt is empty") - planned = records[0] - if not isinstance(planned, dict): - raise RuntimeError("existing repair receipt does not start with valid planned JSON") - planned_keys = {"schema", "state", "target_hash", "targets", "repair_intent_raw_ids", "planned_at_ms"} - if set(planned) != planned_keys or planned.get("schema") != _QUARANTINED_ACCEPTED_RAW_REPAIR_RECEIPT_SCHEMA: - raise RuntimeError("existing repair receipt has an invalid planned record schema") - if planned.get("state") != "planned": - raise RuntimeError("existing repair receipt must start with a planned record") - if planned.get("target_hash") != target_hash or planned.get("targets") != targets: - raise RuntimeError("existing repair receipt targets do not match the proven repair set") - planned_at_ms = planned.get("planned_at_ms") - if not isinstance(planned_at_ms, int) or planned_at_ms < 0: - raise RuntimeError("existing repair receipt planned timestamp is invalid") - raw_ids = tuple(str(target["raw_id"]) for target in targets) - intent = planned.get("repair_intent_raw_ids") - if not isinstance(intent, list) or any(not isinstance(raw_id, str) for raw_id in intent): - raise RuntimeError("existing repair receipt repair intent is invalid") - intent_ids = tuple(cast(list[str], intent)) - if len(set(intent_ids)) != len(intent_ids) or any(raw_id not in raw_ids for raw_id in intent_ids): - raise RuntimeError("existing repair receipt repair intent does not match its targets") - if len(records) == 1: - if not terminated: - raise RuntimeError("existing repair receipt has a torn planned record") - return False, intent_ids, (), terminated - tail = records[1:] - applied = tail[-1] if isinstance(tail[-1], dict) else None - torn_terminals = ( - tuple(record for record in tail[:-1] if isinstance(record, bytes)) - if applied - else tuple(record for record in tail if isinstance(record, bytes)) - ) - expected_tail_length = len(torn_terminals) + (1 if applied is not None else 0) - if len(tail) != expected_tail_length or any(not fragment for fragment in torn_terminals): - raise RuntimeError("existing repair receipt has an invalid state transition") - if applied is None: - return False, intent_ids, torn_terminals, terminated - if not terminated: - raise RuntimeError("existing repair receipt has an unterminated applied record") - recovered = bool(torn_terminals) - applied_keys = { - "schema", - "state", - "target_hash", - "applied_at_ms", - "repaired_raw_ids", - "proven_raw_ids", - } - if recovered: - applied_keys |= {"torn_terminals"} - if set(applied) != applied_keys or applied.get("schema") != _QUARANTINED_ACCEPTED_RAW_REPAIR_RECEIPT_SCHEMA: - raise RuntimeError("existing repair receipt has an invalid applied record schema") - if applied.get("state") != "applied" or applied.get("target_hash") != target_hash: - raise RuntimeError("existing repair receipt has an invalid applied target transition") - applied_at_ms = applied.get("applied_at_ms") - if not isinstance(applied_at_ms, int) or applied_at_ms < 0: - raise RuntimeError("existing repair receipt applied timestamp is invalid") - repaired_ids = applied.get("repaired_raw_ids") - if ( - applied.get("proven_raw_ids") != list(raw_ids) - or not isinstance(repaired_ids, list) - or any(not isinstance(raw_id, str) for raw_id in repaired_ids) - or len(set(cast(list[str], repaired_ids))) != len(repaired_ids) - or any(raw_id not in intent_ids for raw_id in cast(list[str], repaired_ids)) - ): - raise RuntimeError("existing repair receipt applied ids do not match the planned targets") - expected_torn_witnesses = [ - {"bytes": len(fragment), "sha256": hashlib.sha256(fragment).hexdigest()} for fragment in torn_terminals - ] - if recovered and applied.get("torn_terminals") != expected_torn_witnesses: - raise RuntimeError("existing repair receipt recovery does not match its preserved torn terminal") - return True, intent_ids, torn_terminals, terminated - - -def _lock_quarantined_raw_repair_receipt( - path: Path, - items: list[QuarantinedAcceptedRawRepairItem], -) -> _LockedQuarantinedRawRepairReceipt: - """Lock one stable receipt inode and create or validate its planned record.""" - if path.is_symlink(): - raise RuntimeError("repair receipt path must not be a symbolic link") - targets = _repair_receipt_targets(items) - target_hash = hashlib.sha256(json.dumps(targets, sort_keys=True, separators=(",", ":")).encode()).hexdigest() - repair_intent_raw_ids = tuple(item.raw_id for item in items if item.status == "eligible") - flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0) - descriptor = os.open(path, flags, 0o600) - try: - fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError as exc: - os.close(descriptor) - raise RuntimeError("operator repair receipt is already locked by another apply") from exc - try: - opened = os.fstat(descriptor) - named = path.stat(follow_symlinks=False) - if (opened.st_dev, opened.st_ino) != (named.st_dev, named.st_ino): - raise RuntimeError("operator repair receipt path changed while it was being locked") - if opened.st_size: - terminal, existing_intent, torn_terminals, terminated = _validate_repair_receipt_records( - _receipt_records(descriptor), targets=targets, target_hash=target_hash - ) - return _LockedQuarantinedRawRepairReceipt( - path, - descriptor, - target_hash, - terminal, - existing_intent, - torn_terminals, - terminated, - ) - planned = { - "schema": _QUARANTINED_ACCEPTED_RAW_REPAIR_RECEIPT_SCHEMA, - "state": "planned", - "target_hash": target_hash, - "targets": targets, - "repair_intent_raw_ids": list(repair_intent_raw_ids), - "planned_at_ms": int(time.time() * 1000), - } - encoded = (json.dumps(planned, sort_keys=True, separators=(",", ":")) + "\n").encode() - _write_receipt_all(descriptor, encoded) - os.fsync(descriptor) - _fsync_parent(path) - return _LockedQuarantinedRawRepairReceipt(path, descriptor, target_hash, False, repair_intent_raw_ids) - except Exception: - fcntl.flock(descriptor, fcntl.LOCK_UN) - os.close(descriptor) - raise - - -def _finish_quarantined_raw_repair_receipt( - receipt: _LockedQuarantinedRawRepairReceipt, - *, - items: list[QuarantinedAcceptedRawRepairItem], -) -> None: - opened = os.fstat(receipt.descriptor) - named = receipt.path.stat(follow_symlinks=False) - if (opened.st_dev, opened.st_ino) != (named.st_dev, named.st_ino): - raise RuntimeError("operator repair receipt path changed before terminal append") - os.lseek(receipt.descriptor, 0, os.SEEK_END) - preserved_torn_terminals = list(receipt.torn_terminals) - if preserved_torn_terminals and not receipt.receipt_terminated: - # Make even a complete-JSON prefix permanently distinguishable from a - # terminal record after the newline is appended. The exact sealed bytes - # remain in the append-only receipt and are bound into the terminal. - _write_receipt_all(receipt.descriptor, b"\xff\n") - preserved_torn_terminals[-1] += b"\xff" - terminal = { - "schema": _QUARANTINED_ACCEPTED_RAW_REPAIR_RECEIPT_SCHEMA, - "state": "applied", - "target_hash": receipt.target_hash, - "applied_at_ms": int(time.time() * 1000), - "repaired_raw_ids": [item.raw_id for item in items if item.repaired], - "proven_raw_ids": [item.raw_id for item in items], - } - if preserved_torn_terminals: - terminal["torn_terminals"] = [ - {"bytes": len(fragment), "sha256": hashlib.sha256(fragment).hexdigest()} - for fragment in preserved_torn_terminals - ] - _write_receipt_all( - receipt.descriptor, (json.dumps(terminal, sort_keys=True, separators=(",", ":")) + "\n").encode() - ) - os.fsync(receipt.descriptor) - _fsync_parent(receipt.path) - - def _cas_refine_quarantined_accepted_raw( source_conn: sqlite3.Connection, item: QuarantinedAcceptedRawRepairItem, @@ -1391,106 +1111,26 @@ def _cas_refine_quarantined_accepted_raw( raise RuntimeError(f"source authority CAS failed for {item.raw_id}") -def repair_quarantined_accepted_raws( +def inspect_quarantined_accepted_raws( config: Config, raw_ids: list[str], - *, - apply: bool = False, - receipt_path: Path | None = None, - proof_digest: str | None = None, -) -> QuarantinedAcceptedRawRepairReport: - """Refine accepted untyped or typed-quarantined full raws after exact proof.""" +) -> tuple[QuarantinedAcceptedRawRepairItem, ...]: + """Return exact typed quarantine-refinement proofs without mutation.""" if len(set(raw_ids)) != len(raw_ids): raise ValueError("duplicate raw ids are not allowed") if not raw_ids or len(raw_ids) > _QUARANTINED_ACCEPTED_RAW_REPAIR_LIMIT: raise ValueError(f"raw-id list must contain 1..{_QUARANTINED_ACCEPTED_RAW_REPAIR_LIMIT} entries") if any(re.fullmatch(r"[0-9a-f]{64}", raw_id) is None for raw_id in raw_ids): raise ValueError("raw ids must be lowercase SHA-256 identifiers") - block_reason = offline_maintenance_block_reason(config, active=apply, dry_run=not apply) - if block_reason is not None: - raise RuntimeError(block_reason) archive_root = _raw_materialization_archive_root(config) source_db = archive_root / "source.db" index_db = archive_root / "index.db" if not source_db.exists() or not index_db.exists(): raise RuntimeError("source or index tier is missing") - with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as dry_conn: - _attach_repair_index(dry_conn, index_db) - _validate_quarantined_raw_repair_blob_budget(dry_conn, raw_ids) - items = [_inspect_quarantined_accepted_raw(archive_root, raw_id, conn=dry_conn) for raw_id in raw_ids] - aggregate_proof = _repair_proof_digest(items) - if apply and any(item.status == "ineligible" for item in items): - raise RuntimeError("quarantined accepted raw repair refused because one or more targets are ineligible") - if apply and receipt_path is None: - raise ValueError("apply requires an explicit operator repair receipt path") - if apply and proof_digest != aggregate_proof: - raise RuntimeError("apply proof digest does not match the exact dry-run target list") - if apply: - from polylogue.storage.index_generation import RebuildLease - - assert receipt_path is not None - with RebuildLease(archive_root): - receipt = _lock_quarantined_raw_repair_receipt(receipt_path, items) - try: - with closing(sqlite3.connect(f"file:{source_db}?mode=rw", uri=True)) as source_conn: - source_conn.execute("PRAGMA foreign_keys = ON") - _attach_repair_index(source_conn, index_db) - source_conn.execute("BEGIN IMMEDIATE") - try: - _validate_quarantined_raw_repair_blob_budget(source_conn, raw_ids) - locked_items = [ - _inspect_quarantined_accepted_raw(archive_root, raw_id, conn=source_conn) - for raw_id in raw_ids - ] - if _repair_proof_digest(locked_items) != proof_digest: - raise RuntimeError("authority proof changed after acquiring the repair transaction") - if any(item.status == "ineligible" for item in locked_items): - raise RuntimeError("a repair target became ineligible after acquiring the transaction") - if receipt.terminal and any(item.status != "already_repaired" for item in locked_items): - raise RuntimeError("terminal operator receipt disagrees with durable source authority") - if receipt.torn_terminals and any(item.status != "already_repaired" for item in locked_items): - raise RuntimeError("torn terminal receipt has no matching committed source refinement") - for item in locked_items: - if item.status == "eligible" and item.census_stage_raw_ids: - _stage_quarantined_census_cohort(source_conn, item) - staged_items = [ - _inspect_quarantined_accepted_raw(archive_root, raw_id, conn=source_conn) - for raw_id in raw_ids - ] - if any(item.status == "ineligible" for item in staged_items): - raise RuntimeError("census staging did not preserve the proven repair shape") - for item in staged_items: - if item.status == "eligible": - _cas_refine_quarantined_accepted_raw(source_conn, item) - after_items = [ - _inspect_quarantined_accepted_raw(archive_root, raw_id, conn=source_conn) - for raw_id in raw_ids - ] - if any(item.status != "already_repaired" for item in after_items): - raise RuntimeError("source envelope refinement did not reach the proven terminal state") - source_conn.commit() - except Exception: - source_conn.rollback() - raise - items = [ - dataclasses.replace(after, repaired=before.status == "eligible") - for before, after in zip(locked_items, after_items, strict=True) - ] - if not receipt.terminal: - _finish_quarantined_raw_repair_receipt(receipt, items=items) - finally: - receipt.close() - return QuarantinedAcceptedRawRepairReport( - mode="apply" if apply else "dry-run", - requested_count=len(items), - eligible_count=sum(item.status == "eligible" for item in items), - repaired_count=sum(item.repaired for item in items), - already_repaired_count=sum(item.status == "already_repaired" for item in items), - ineligible_count=sum(item.status == "ineligible" for item in items), - proof_digest=aggregate_proof, - receipt_path=str(receipt_path) if receipt_path is not None else None, - items=tuple(items), - ) + with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as conn: + _attach_repair_index(conn, index_db) + _validate_quarantined_raw_repair_blob_budget(conn, raw_ids) + return tuple(_inspect_quarantined_accepted_raw(archive_root, raw_id, conn=conn) for raw_id in raw_ids) def _browser_origin_ineligible(raw_id: str, reason: str) -> BrowserCaptureOriginRepairItem: @@ -1498,24 +1138,13 @@ def _browser_origin_ineligible(raw_id: str, reason: str) -> BrowserCaptureOrigin def _browser_origin_item_payload(item: BrowserCaptureOriginRepairItem) -> dict[str, object]: - excluded = { - "status", - "reason", - "proof_digest", - "repaired", - "copy_forward_source_complete", - "terminal_byte_witness_digest", - } - # These fields belong solely to the separately versioned legacy actuator. - # Keeping them out of ordinary payloads preserves pre-legacy v1 receipt - # targets exactly, so a planned ordinary repair can still resume. - if not item.legacy_null_native_id: - excluded.update({"legacy_null_native_id", "parser_derived_native_id"}) - if not item.byte_proven_null_native_id_rekey: - excluded.update({"byte_proven_null_native_id_rekey", "parsed_message_count"}) + """Canonical exact strategy witness for the shared raw-authority plan.""" + # ``copy_forward_source_complete`` is an execution checkpoint: it flips + # after the source row is staged but before the index CAS completes. It + # therefore cannot be part of the immutable strategy identity. The + # actual copy/raw footprint and terminal byte witness remain bound. + excluded = {"status", "reason", "proof_digest", "repaired", "copy_forward_source_complete"} payload = {key: value for key, value in dataclasses.asdict(item).items() if key not in excluded} - # Receipts are JSONL. Normalize tuples now so an in-memory item has the - # identical shape when it is compared to a re-read planned record. return cast(dict[str, object], json.loads(json.dumps(payload, sort_keys=True, separators=(",", ":")))) @@ -1525,49 +1154,6 @@ def _browser_origin_item_digest(item: BrowserCaptureOriginRepairItem) -> str: ).hexdigest() -def _browser_origin_proof_digest(items: list[BrowserCaptureOriginRepairItem]) -> str: - return hashlib.sha256(json.dumps([item.proof_digest for item in items], separators=(",", ":")).encode()).hexdigest() - - -def _browser_origin_semantic_witness_bindings(items: list[BrowserCaptureOriginRepairItem]) -> list[dict[str, object]]: - return [ - { - "raw_id": item.raw_id, - "semantic_canonical_raw_id": item.semantic_canonical_raw_id, - "semantic_historical_raw_ids": list(item.semantic_historical_raw_ids), - "semantic_head_snapshot": item.semantic_head_snapshot, - "semantic_witness_digest": item.semantic_witness_digest, - } - for item in items - ] - - -def _browser_origin_terminal_byte_witness_bindings( - items: list[BrowserCaptureOriginRepairItem], -) -> list[dict[str, object]]: - return [ - { - "raw_id": item.raw_id, - "copy_forward_raw_id": item.copy_forward_raw_id, - "terminal_byte_witness_digest": item.terminal_byte_witness_digest, - } - for item in items - ] - - -def _browser_origin_legacy_native_witness_bindings( - items: list[BrowserCaptureOriginRepairItem], -) -> list[dict[str, object]]: - return [ - { - "raw_id": item.raw_id, - "legacy_null_native_id": item.legacy_null_native_id, - "parser_derived_native_id": item.parser_derived_native_id, - } - for item in items - ] - - def _browser_origin_copy_forward_detail(item: BrowserCaptureOriginRepairItem) -> str: """Encode the proven semantic-head snapshot in the immutable copy receipt.""" payload: dict[str, object] = { @@ -2797,6 +2383,7 @@ def _inspect_browser_capture_origin_mismatch( repair_strategy=repair_strategy, replacement_raw_id=replacement_raw_id, replacement_source_revision=replacement_source_revision, + replacement_content_hash=accepted_hash.hex(), replacement_frontier_kind=replacement_frontier_kind, replacement_frontier=replacement_frontier, copy_forward_raw_id=copy_raw_id if repair_strategy == "copy_forward" else None, @@ -2815,271 +2402,6 @@ def _inspect_browser_capture_origin_mismatch( return dataclasses.replace(item, proof_digest=_browser_origin_item_digest(item)) -@dataclass(slots=True) -class _BrowserOriginReceipt: - path: Path - descriptor: int - target_hash: str - terminal: bool - torn: bytes = b"" - - def close(self) -> None: - fcntl.flock(self.descriptor, fcntl.LOCK_UN) - os.close(self.descriptor) - - -def _browser_origin_requires_legacy_transaction(items: list[BrowserCaptureOriginRepairItem]) -> bool: - return any(item.legacy_null_native_id or item.byte_proven_null_native_id_rekey for item in items) - - -def _legacy_browser_journal_modes(source_db: Path, index_db: Path) -> dict[str, str]: - modes: dict[str, str] = {} - for name, db in (("source", source_db), ("index", index_db)): - with closing(sqlite3.connect(f"file:{db}?mode=rw", uri=True, timeout=30.0)) as conn: - conn.execute("PRAGMA busy_timeout = 30000") - row = conn.execute("PRAGMA journal_mode").fetchone() - if row is None: - raise RuntimeError(f"legacy browser repair could not read {name} journal mode") - modes[name] = str(row[0]).lower() - return modes - - -def _legacy_browser_set_journal_mode(db: Path, *, name: str, mode: str) -> None: - if mode not in {"delete", "wal"}: - raise ValueError(f"unsupported legacy browser journal mode: {mode}") - with closing(sqlite3.connect(f"file:{db}?mode=rw", uri=True, timeout=30.0)) as conn: - conn.execute("PRAGMA busy_timeout = 30000") - row = conn.execute(f"PRAGMA journal_mode = {mode.upper()}").fetchone() - if row is None or str(row[0]).lower() != mode: - raise RuntimeError(f"legacy browser repair could not set {name} journal_mode={mode}") - - -def _legacy_browser_checkpoint_wal(db: Path, *, name: str) -> None: - with closing(sqlite3.connect(f"file:{db}?mode=rw", uri=True, timeout=30.0)) as conn: - conn.execute("PRAGMA busy_timeout = 30000") - row = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() - if row is None or tuple(int(value) for value in row) != (0, 0, 0): - raise RuntimeError(f"legacy browser repair requires an idle {name} WAL checkpoint: {row!r}") - - -def _legacy_browser_validate_atomic_file_set(source_db: Path, index_db: Path) -> None: - source = source_db.resolve(strict=True) - index = index_db.resolve(strict=True) - if source.stat().st_dev != index.stat().st_dev: - raise RuntimeError("legacy browser repair requires source.db and index.db in one archive filesystem") - - -def _legacy_browser_normalize_journal_posture(source_db: Path, index_db: Path) -> dict[str, str]: - """Recover any prior legacy interruption to the normal WAL serving posture.""" - before = _legacy_browser_journal_modes(source_db, index_db) - unsupported = {name: mode for name, mode in before.items() if mode not in {"wal", "delete"}} - if unsupported: - raise RuntimeError(f"legacy browser repair found unsupported journal posture: {unsupported!r}") - for name, db in (("source", source_db), ("index", index_db)): - if before[name] != "wal": - _legacy_browser_set_journal_mode(db, name=name, mode="wal") - after = _legacy_browser_journal_modes(source_db, index_db) - if after != {"source": "wal", "index": "wal"}: - raise RuntimeError(f"legacy browser repair could not restore WAL posture: {after!r}") - return before - - -def _legacy_browser_copy_forward_checkpoint(stage: str) -> None: - """Test seam for a process death at an exact crash boundary.""" - del stage - - -@contextmanager -def _legacy_browser_rollback_superjournal_window( - source_db: Path, index_db: Path -) -> Iterator[tuple[sqlite3.Connection, dict[str, dict[str, str]]]]: - """Yield the legacy-only crash-atomic cross-tier maintenance transaction.""" - _legacy_browser_validate_atomic_file_set(source_db, index_db) - observation: dict[str, dict[str, str]] = {"before_transport": _legacy_browser_journal_modes(source_db, index_db)} - conn: sqlite3.Connection | None = None - failed = False - try: - _legacy_browser_checkpoint_wal(index_db, name="index") - _legacy_browser_checkpoint_wal(source_db, name="source") - _legacy_browser_set_journal_mode(index_db, name="index", mode="delete") - _legacy_browser_set_journal_mode(source_db, name="source", mode="delete") - if _legacy_browser_journal_modes(source_db, index_db) != {"source": "delete", "index": "delete"}: - raise RuntimeError("legacy browser repair could not enter rollback-journal posture") - conn = sqlite3.connect(f"file:{index_db}?mode=rw", uri=True, timeout=30.0) - conn.execute("PRAGMA busy_timeout = 30000") - conn.execute("PRAGMA foreign_keys = ON") - conn.execute("ATTACH DATABASE ? AS source", (str(source_db),)) - conn.execute("PRAGMA main.synchronous = EXTRA") - conn.execute("PRAGMA source.synchronous = EXTRA") - if int(conn.execute("PRAGMA main.synchronous").fetchone()[0]) != 3: - raise RuntimeError("legacy browser repair could not set index synchronous=EXTRA") - if int(conn.execute("PRAGMA source.synchronous").fetchone()[0]) != 3: - raise RuntimeError("legacy browser repair could not set source synchronous=EXTRA") - conn.execute("BEGIN EXCLUSIVE") - yield conn, observation - except BaseException: - failed = True - if conn is not None and conn.in_transaction: - with suppress(sqlite3.Error): - conn.rollback() - raise - finally: - if conn is not None: - conn.close() - try: - # A completed or rolled-back transport always returns the daemon's - # normal WAL posture before its receipt can become terminal. - _legacy_browser_set_journal_mode(source_db, name="source", mode="wal") - _legacy_browser_set_journal_mode(index_db, name="index", mode="wal") - observation["after_restore"] = _legacy_browser_journal_modes(source_db, index_db) - if observation["after_restore"] != {"source": "wal", "index": "wal"}: - raise RuntimeError("legacy browser repair did not restore WAL posture") - except Exception: - if failed: - logger.exception("legacy browser repair could not restore WAL after a failed transport") - else: - raise - - -def _lock_browser_origin_receipt( - path: Path, - items: list[BrowserCaptureOriginRepairItem], - *, - schema: str, -) -> _BrowserOriginReceipt: - if path.is_symlink(): - raise RuntimeError("repair receipt path must not be a symbolic link") - targets = [_browser_origin_item_payload(item) for item in items] - requires_legacy_transaction = _browser_origin_requires_legacy_transaction(items) - target_hash = hashlib.sha256(json.dumps(targets, sort_keys=True, separators=(",", ":")).encode()).hexdigest() - descriptor = os.open(path, os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600) - try: - fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) - opened = os.fstat(descriptor) - named = path.stat(follow_symlinks=False) - if (opened.st_dev, opened.st_ino) != (named.st_dev, named.st_ino): - raise RuntimeError("repair receipt path changed while it was being locked") - os.lseek(descriptor, 0, os.SEEK_SET) - chunks: list[bytes] = [] - remaining = opened.st_size - while remaining: - chunk = os.read(descriptor, remaining) - if not chunk: - raise RuntimeError("existing repair receipt changed during its locked read") - chunks.append(chunk) - remaining -= len(chunk) - existing = b"".join(chunks) - if existing: - first, separator, tail = existing.partition(b"\n") - if not separator: - raise RuntimeError("existing repair receipt has a torn planned record") - try: - planned = json.loads(first) - except (ValueError, json.JSONDecodeError) as exc: - raise RuntimeError("existing repair receipt has invalid planned JSON") from exc - expected = { - "schema": schema, - "state": "planned", - "target_hash": target_hash, - "targets": targets, - } - if requires_legacy_transaction: - expected["transaction_protocol"] = _LEGACY_BROWSER_NATIVE_ID_TRANSACTION_PROTOCOL - if {key: planned.get(key) for key in expected} != expected: - raise RuntimeError("existing repair receipt targets do not match the exact proof") - terminal = False - torn = tail - if tail.endswith(b"\n") and tail: - lines = tail.splitlines() - try: - applied = json.loads(lines[-1]) - except (ValueError, json.JSONDecodeError): - applied = None - if isinstance(applied, dict) and applied.get("state") == "applied": - requires_legacy_witness = requires_legacy_transaction - legacy_witness_matches = applied.get( - "legacy_native_witness_bindings" - ) == _browser_origin_legacy_native_witness_bindings(items) - if ( - applied.get("schema") != schema - or applied.get("target_hash") != target_hash - or applied.get("replacement_raw_ids") != [item.replacement_raw_id for item in items] - or applied.get("semantic_witness_bindings") != _browser_origin_semantic_witness_bindings(items) - or applied.get("terminal_byte_witness_bindings") - != _browser_origin_terminal_byte_witness_bindings(items) - or (requires_legacy_witness and not legacy_witness_matches) - or ( - requires_legacy_witness - and applied.get("transaction_protocol") != _LEGACY_BROWSER_NATIVE_ID_TRANSACTION_PROTOCOL - ) - or ( - not requires_legacy_witness - and applied.get("legacy_native_witness_bindings") is not None - and not legacy_witness_matches - ) - ): - raise RuntimeError("existing repair receipt has a conflicting terminal record") - terminal = True - torn = b"" - return _BrowserOriginReceipt(path, descriptor, target_hash, terminal, torn) - planned = { - "schema": schema, - "state": "planned", - "target_hash": target_hash, - "targets": targets, - "planned_at_ms": int(time.time() * 1000), - } - if requires_legacy_transaction: - planned["transaction_protocol"] = _LEGACY_BROWSER_NATIVE_ID_TRANSACTION_PROTOCOL - _write_receipt_all(descriptor, (json.dumps(planned, sort_keys=True, separators=(",", ":")) + "\n").encode()) - os.fsync(descriptor) - _fsync_parent(path) - return _BrowserOriginReceipt(path, descriptor, target_hash, False) - except Exception: - with suppress(OSError): - fcntl.flock(descriptor, fcntl.LOCK_UN) - os.close(descriptor) - raise - - -def _finish_browser_origin_receipt( - receipt: _BrowserOriginReceipt, - items: list[BrowserCaptureOriginRepairItem], - *, - schema: str, - legacy_journal_modes: Mapping[str, object] | None = None, -) -> None: - os.lseek(receipt.descriptor, 0, os.SEEK_END) - terminal: dict[str, object] = { - "schema": schema, - "state": "applied", - "target_hash": receipt.target_hash, - "applied_at_ms": int(time.time() * 1000), - "replacement_raw_ids": [item.replacement_raw_id for item in items], - "semantic_witness_bindings": _browser_origin_semantic_witness_bindings(items), - "terminal_byte_witness_bindings": _browser_origin_terminal_byte_witness_bindings(items), - "legacy_native_witness_bindings": _browser_origin_legacy_native_witness_bindings(items), - } - if _browser_origin_requires_legacy_transaction(items): - if legacy_journal_modes is None: - raise RuntimeError("legacy browser repair cannot finalize without journal protocol evidence") - terminal["transaction_protocol"] = _LEGACY_BROWSER_NATIVE_ID_TRANSACTION_PROTOCOL - terminal["legacy_journal_modes"] = dict(legacy_journal_modes) - if receipt.torn: - if not receipt.torn.endswith(b"\n"): - _write_receipt_all(receipt.descriptor, b"\xff\n") - terminal["preserved_torn_terminal"] = { - "bytes": len(receipt.torn), - "sha256": hashlib.sha256(receipt.torn).hexdigest(), - } - _write_receipt_all( - receipt.descriptor, - (json.dumps(terminal, sort_keys=True, separators=(",", ":")) + "\n").encode(), - ) - os.fsync(receipt.descriptor) - _fsync_parent(receipt.path) - - def _stage_browser_origin_copy_forward_source( conn: sqlite3.Connection, item: BrowserCaptureOriginRepairItem, @@ -3268,14 +2590,80 @@ def _finalize_browser_origin_copy_forward_index(conn: sqlite3.Connection, item: ), decided_at_ms=acquired_at_ms, ) + _retire_browser_origin_legacy_head( + conn, + item, + accepted_raw_id=item.copy_forward_raw_id, + accepted_source_revision=item.blob_hash, + accepted_content_hash=item.replacement_content_hash or item.accepted_content_hash, + accepted_frontier_kind="byte", + accepted_frontier=item.accepted_frontier, + decided_at_ms=acquired_at_ms, + ) -def _restore_browser_origin_canonical_head(conn: sqlite3.Connection, item: BrowserCaptureOriginRepairItem) -> None: +def _retire_browser_origin_legacy_head( + conn: sqlite3.Connection, + item: BrowserCaptureOriginRepairItem, + *, + accepted_raw_id: str, + accepted_source_revision: str, + accepted_content_hash: str, + accepted_frontier_kind: str, + accepted_frontier: int, + decided_at_ms: int, +) -> None: + """Terminally receipt and remove the exact obsolete unknown-key head.""" from polylogue.storage.sqlite.archive_tiers.revision_application import ( RevisionApplicationReceipt, record_revision_application_sync, ) + assert item.session_id is not None + assert item.old_logical_source_key is not None + assert item.blob_hash is not None + assert item.accepted_content_hash is not None + assert item.accepted_frontier is not None + record_revision_application_sync( + conn, + RevisionApplicationReceipt( + raw_id=item.raw_id, + session_id=item.session_id, + logical_source_key=item.old_logical_source_key, + source_revision=item.blob_hash, + acquisition_generation=0, + decision=ApplicationDecision.SUPERSEDED, + accepted_raw_id=accepted_raw_id, + accepted_source_revision=accepted_source_revision, + accepted_content_hash=bytes.fromhex(accepted_content_hash), + accepted_frontier_kind=accepted_frontier_kind, + accepted_frontier=accepted_frontier, + detail=f"browser_capture_origin_supersession:{item.raw_id}", + ), + decided_at_ms=decided_at_ms, + ) + deleted = conn.execute( + """ + DELETE FROM raw_revision_heads + WHERE logical_source_key = ? AND session_id = ? AND accepted_raw_id = ? + AND accepted_source_revision = ? AND accepted_content_hash = ? + AND accepted_frontier_kind = 'byte' AND accepted_frontier = ? + AND acquisition_generation = 0 AND append_end_offset IS NULL + """, + ( + item.old_logical_source_key, + item.session_id, + item.raw_id, + item.blob_hash, + bytes.fromhex(item.accepted_content_hash), + item.accepted_frontier, + ), + ).rowcount + if deleted != 1: + raise RuntimeError(f"obsolete browser-origin head CAS failed for {item.raw_id}") + + +def _restore_browser_origin_canonical_head(conn: sqlite3.Connection, item: BrowserCaptureOriginRepairItem) -> None: assert item.session_id is not None assert item.canonical_logical_source_key is not None assert item.blob_hash is not None @@ -3291,22 +2679,14 @@ def _restore_browser_origin_canonical_head(conn: sqlite3.Connection, item: Brows ) if cursor.rowcount != 1: raise RuntimeError(f"session raw pointer CAS failed for {item.raw_id}") - record_revision_application_sync( + _retire_browser_origin_legacy_head( conn, - RevisionApplicationReceipt( - raw_id=item.raw_id, - session_id=item.session_id, - logical_source_key=item.canonical_logical_source_key, - source_revision=item.blob_hash, - acquisition_generation=0, - decision=ApplicationDecision.SUPERSEDED, - accepted_raw_id=item.replacement_raw_id, - accepted_source_revision=item.replacement_source_revision, - accepted_content_hash=bytes.fromhex(item.accepted_content_hash), - accepted_frontier_kind=item.replacement_frontier_kind, - accepted_frontier=item.replacement_frontier, - detail=f"browser_capture_origin_supersession:{item.raw_id}", - ), + item, + accepted_raw_id=item.replacement_raw_id, + accepted_source_revision=item.replacement_source_revision, + accepted_content_hash=item.replacement_content_hash or item.accepted_content_hash, + accepted_frontier_kind=item.replacement_frontier_kind, + accepted_frontier=item.replacement_frontier, decided_at_ms=decided_at_ms, ) @@ -3321,272 +2701,179 @@ def _apply_browser_origin_repair_item(conn: sqlite3.Connection, item: BrowserCap raise RuntimeError(f"unsupported browser-capture origin repair strategy: {item.repair_strategy}") -def _repair_browser_capture_origin_mismatches( +def _browser_origin_strategy_terminal(conn: sqlite3.Connection, item: BrowserCaptureOriginRepairItem) -> bool: + """Prove the shared strategy retired the legacy head and selected its successor.""" + replacement_raw_id = item.copy_forward_raw_id or item.replacement_raw_id + if ( + item.session_id is None + or item.old_logical_source_key is None + or item.canonical_logical_source_key is None + or replacement_raw_id is None + or item.replacement_source_revision is None + or item.replacement_frontier_kind is None + or item.replacement_frontier is None + ): + return False + session = conn.execute("SELECT raw_id FROM sessions WHERE session_id = ?", (item.session_id,)).fetchone() + canonical = conn.execute( + """ + SELECT accepted_raw_id, accepted_source_revision, accepted_content_hash, + accepted_frontier_kind, accepted_frontier + FROM raw_revision_heads WHERE logical_source_key = ? AND session_id = ? + """, + (item.canonical_logical_source_key, item.session_id), + ).fetchone() + legacy = conn.execute( + "SELECT 1 FROM raw_revision_heads WHERE logical_source_key = ?", + (item.old_logical_source_key,), + ).fetchone() + superseded = conn.execute( + """ + SELECT accepted_raw_id, accepted_source_revision, accepted_content_hash + FROM raw_revision_applications + WHERE raw_id = ? AND session_id = ? AND logical_source_key = ? + AND decision = 'superseded' + """, + (item.raw_id, item.session_id, item.old_logical_source_key), + ).fetchone() + expected = ( + replacement_raw_id, + item.replacement_source_revision, + bytes.fromhex(item.replacement_content_hash or item.accepted_content_hash or ""), + item.replacement_frontier_kind, + item.replacement_frontier, + ) + expected_supersession = expected[:3] + return ( + session is not None + and str(session[0]) == replacement_raw_id + and canonical is not None + and tuple(canonical) == expected + and legacy is None + and superseded is not None + and tuple(superseded) == expected_supersession + ) + + +def _apply_browser_conflict_canonical_resolution( + archive_root: Path, + conn: sqlite3.Connection, + raw_id: str, + expected_witness: Mapping[str, object], +) -> None: + """Apply one explicit retain-canonical judgment against an exact conflict witness.""" + base = _inspect_browser_capture_origin_strategy(archive_root, raw_id, conn=conn) + if base.status != "ineligible": + raise RuntimeError("conflict resolution no longer observes a conflicting browser authority") + current = _browser_canonical_authority_conflict_witness(archive_root, conn, raw_id, base.reason) + current_witness = dataclasses.asdict(current) + # ``reason`` is the inspector's human-readable rejection path. It can + # legitimately become more specific as the shared classifier evolves and + # is deliberately excluded from the evidence digest. Every structural + # witness field remains an exact CAS precondition. + comparable_current = {key: value for key, value in current_witness.items() if key != "reason"} + comparable_expected = {key: value for key, value in expected_witness.items() if key != "reason"} + if comparable_current != comparable_expected: + changed_fields = sorted( + key + for key in comparable_current.keys() | comparable_expected.keys() + if comparable_current.get(key) != comparable_expected.get(key) + ) + raise RuntimeError( + f"browser conflict evidence changed after operator judgment: fields={','.join(changed_fields)}" + ) + required = ( + current.session_id, + current.old_logical_source_key, + current.canonical_logical_source_key, + current.unknown_raw_content_hash, + current.unknown_source_revision, + current.unknown_frontier, + current.competing_raw_id, + current.competing_content_hash, + current.competing_source_revision, + current.competing_frontier_kind, + current.competing_frontier, + ) + if any(value is None for value in required): + raise RuntimeError("retain-canonical judgment lacks a complete typed competing-head witness") + repair_item = BrowserCaptureOriginRepairItem( + raw_id=raw_id, + status="eligible", + reason="accepted operator judgment retained canonical authority", + session_id=current.session_id, + old_logical_source_key=current.old_logical_source_key, + canonical_logical_source_key=current.canonical_logical_source_key, + blob_hash=current.unknown_source_revision, + accepted_content_hash=current.unknown_raw_content_hash, + accepted_frontier=current.unknown_frontier, + repair_strategy="restore_canonical_head", + replacement_raw_id=current.competing_raw_id, + replacement_source_revision=current.competing_source_revision, + replacement_content_hash=current.competing_content_hash, + replacement_frontier_kind=current.competing_frontier_kind, + replacement_frontier=current.competing_frontier, + ) + _restore_browser_origin_canonical_head(conn, repair_item) + if not _browser_origin_strategy_terminal(conn, repair_item): + raise RuntimeError("retain-canonical judgment did not reach its typed terminal postcondition") + + +def _inspect_browser_capture_origin_strategy( + archive_root: Path, + raw_id: str, + *, + conn: sqlite3.Connection, +) -> BrowserCaptureOriginRepairItem: + """Select one admitted strategy shape from its durable source envelope.""" + conn.row_factory = sqlite3.Row + envelope = conn.execute( + "SELECT native_id, revision_authority FROM source.raw_sessions WHERE raw_id = ?", + (raw_id,), + ).fetchone() + legacy_null = bool( + envelope is not None + and envelope["native_id"] is None + and envelope["revision_authority"] == RawRevisionAuthority.QUARANTINED.value + ) + byte_proven_null = bool( + envelope is not None + and envelope["native_id"] is None + and envelope["revision_authority"] == RawRevisionAuthority.BYTE_PROVEN.value + ) + return _inspect_browser_capture_origin_mismatch( + archive_root, + raw_id, + conn=conn, + allow_legacy_null_native_id=legacy_null, + allow_byte_proven_null_native_id_rekey=byte_proven_null, + ) + + +def inspect_browser_capture_origin_mismatches( config: Config, raw_ids: list[str], - *, - apply: bool = False, - receipt_path: Path | None = None, - proof_digest: str | None = None, - allow_legacy_null_native_id: bool = False, - allow_byte_proven_null_native_id_rekey: bool = False, - receipt_schema: str = _BROWSER_CAPTURE_ORIGIN_REPAIR_RECEIPT_SCHEMA, -) -> BrowserCaptureOriginRepairReport: - """Copy exact browser-capture evidence forward under parsed origin authority.""" +) -> tuple[BrowserCaptureOriginRepairItem, ...]: + """Return the exact admitted browser-origin strategy for each raw. + + The durable source envelope selects the applicable strategy shape. This + keeps historical null-native-id variants behind the same inspector and + plan contract instead of exposing mode-specific repair entrypoints. + """ if len(set(raw_ids)) != len(raw_ids): raise ValueError("duplicate raw ids are not allowed") if not raw_ids or len(raw_ids) > _QUARANTINED_ACCEPTED_RAW_REPAIR_LIMIT: raise ValueError("raw-id list must contain 1..100 entries") if any(re.fullmatch(r"[0-9a-f]{64}", raw_id) is None for raw_id in raw_ids): raise ValueError("raw ids must be lowercase SHA-256 identifiers") - block_reason = offline_maintenance_block_reason(config, active=apply, dry_run=not apply) - if block_reason is not None: - raise RuntimeError(block_reason) archive_root = _raw_materialization_archive_root(config) source_db = archive_root / "source.db" index_db = archive_root / "index.db" if not source_db.exists() or not index_db.exists(): raise RuntimeError("source or index tier is missing") - - def inspect(connection: sqlite3.Connection) -> list[BrowserCaptureOriginRepairItem]: - return [ - _inspect_browser_capture_origin_mismatch( - archive_root, - raw_id, - conn=connection, - allow_legacy_null_native_id=allow_legacy_null_native_id, - allow_byte_proven_null_native_id_rekey=allow_byte_proven_null_native_id_rekey, - ) - for raw_id in raw_ids - ] - with closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True)) as conn: conn.execute("ATTACH DATABASE ? AS source", (str(source_db),)) - items = inspect(conn) - aggregate = _browser_origin_proof_digest(items) - if apply and any(item.status == "ineligible" for item in items): - raise RuntimeError("browser-capture origin repair refused because one or more targets are ineligible") - if apply and receipt_path is None: - raise ValueError("apply requires an explicit operator repair receipt path") - if apply and proof_digest != aggregate: - raise RuntimeError("apply proof digest does not match the exact dry-run target list") - if apply: - from polylogue.storage.blob_publication import exclude_archive_blob_publishers - from polylogue.storage.index_generation import RebuildLease - - assert receipt_path is not None - receipt_path.parent.mkdir(parents=True, exist_ok=True) - with RebuildLease(archive_root), exclude_archive_blob_publishers(source_db): - receipt = _lock_browser_origin_receipt(receipt_path, items, schema=receipt_schema) - try: - legacy_journal_modes: Mapping[str, object] | None = None - if _browser_origin_requires_legacy_transaction(items): - # A previous process may have died after its crash-atomic - # commit and before WAL restoration/receipt finalization. - # Normalize that safe mixed posture before inspecting any - # terminal state or issuing another legacy transaction. - legacy_journal_modes = { - "before_normalization": _legacy_browser_normalize_journal_posture(source_db, index_db) - } - with closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True)) as proof_conn: - proof_conn.execute("ATTACH DATABASE ? AS source", (str(source_db),)) - pre_source_items = inspect(proof_conn) - if _browser_origin_proof_digest(pre_source_items) != proof_digest: - raise RuntimeError("authority proof changed before copy-forward source staging") - if any(item.status == "ineligible" for item in pre_source_items): - raise RuntimeError("a browser-capture origin target became ineligible before source staging") - if _browser_origin_requires_legacy_transaction(pre_source_items): - if all(item.status == "already_repaired" for item in pre_source_items): - locked = pre_source_items - after = pre_source_items - assert legacy_journal_modes is not None - legacy_journal_modes = { - **legacy_journal_modes, - "after_restore": _legacy_browser_journal_modes(source_db, index_db), - } - else: - # The legacy route must never leave a source-only - # stage. DELETE rollback journals plus an attached - # non-memory index main database let SQLite use its - # crash-atomic multi-file super-journal protocol. - with _legacy_browser_rollback_superjournal_window(source_db, index_db) as ( - conn, - transport_journal_modes, - ): - locked = inspect(conn) - if _browser_origin_proof_digest(locked) != proof_digest: - raise RuntimeError("authority proof changed after acquiring the repair transaction") - if any(item.status == "ineligible" for item in locked): - raise RuntimeError("a browser-capture origin target became ineligible") - if receipt.terminal and any(item.status != "already_repaired" for item in locked): - raise RuntimeError( - "terminal operator receipt disagrees with durable copy-forward state" - ) - for item in locked: - if ( - item.status == "eligible" - and item.repair_strategy == "copy_forward" - and not item.copy_forward_source_complete - ): - _verify_browser_origin_copy_forward_source_stage( - archive_root, conn, item, source_schema="source" - ) - _stage_browser_origin_copy_forward_source(conn, item, source_schema="source") - for item in locked: - if item.status == "eligible": - _apply_browser_origin_repair_item(conn, item) - after = inspect(conn) - if any(item.status != "already_repaired" for item in after): - raise RuntimeError("legacy browser copy-forward did not reach terminal state") - _legacy_browser_copy_forward_checkpoint("before_commit") - conn.commit() - _legacy_browser_copy_forward_checkpoint("after_commit") - with closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True)) as postflight_conn: - postflight_conn.execute("ATTACH DATABASE ? AS source", (str(source_db),)) - after = inspect(postflight_conn) - if any(item.status != "already_repaired" for item in after): - raise RuntimeError("legacy browser copy-forward did not survive WAL restoration") - assert legacy_journal_modes is not None - legacy_journal_modes = { - **legacy_journal_modes, - **transport_journal_modes, - } - if receipt.terminal and any(item.status != "already_repaired" for item in after): - raise RuntimeError("terminal operator receipt disagrees with durable copy-forward state") - else: - with closing(sqlite3.connect(f"file:{source_db}?mode=rw", uri=True)) as source_conn: - source_conn.execute("PRAGMA foreign_keys = ON") - source_conn.execute("BEGIN IMMEDIATE") - try: - for item in items: - if ( - item.status == "eligible" - and item.repair_strategy == "copy_forward" - and not item.copy_forward_source_complete - ): - _verify_browser_origin_copy_forward_source_stage(archive_root, source_conn, item) - _stage_browser_origin_copy_forward_source(source_conn, item) - source_conn.commit() - except Exception: - source_conn.rollback() - raise - with closing(sqlite3.connect(f"file:{index_db}?mode=rw", uri=True)) as conn: - conn.execute("PRAGMA foreign_keys = ON") - conn.execute("ATTACH DATABASE ? AS source", (str(source_db),)) - conn.execute("BEGIN IMMEDIATE") - try: - locked = inspect(conn) - if _browser_origin_proof_digest(locked) != proof_digest: - raise RuntimeError("authority proof changed after acquiring the repair transaction") - if any(item.status == "ineligible" for item in locked): - raise RuntimeError("a browser-capture origin target became ineligible") - if receipt.terminal and any(item.status != "already_repaired" for item in locked): - raise RuntimeError( - "terminal operator receipt disagrees with durable copy-forward state" - ) - for item in locked: - if item.status == "eligible": - _apply_browser_origin_repair_item(conn, item) - after = inspect(conn) - if any(item.status != "already_repaired" for item in after): - raise RuntimeError("browser-capture origin copy-forward did not reach terminal state") - conn.commit() - except Exception: - conn.rollback() - raise - items = [ - dataclasses.replace(after_item, repaired=before.status == "eligible") - for before, after_item in zip(locked, after, strict=True) - ] - if not receipt.terminal: - _finish_browser_origin_receipt( - receipt, - items, - schema=receipt_schema, - legacy_journal_modes=legacy_journal_modes, - ) - finally: - receipt.close() - return BrowserCaptureOriginRepairReport( - mode="apply" if apply else "dry-run", - requested_count=len(items), - eligible_count=sum(item.status == "eligible" for item in items), - repaired_count=sum(item.repaired for item in items), - already_repaired_count=sum(item.status == "already_repaired" for item in items), - ineligible_count=sum(item.status == "ineligible" for item in items), - proof_digest=aggregate, - receipt_path=str(receipt_path) if receipt_path is not None else None, - items=tuple(items), - ) - - -def repair_browser_capture_origin_mismatches( - config: Config, - raw_ids: list[str], - *, - apply: bool = False, - receipt_path: Path | None = None, - proof_digest: str | None = None, -) -> BrowserCaptureOriginRepairReport: - """Copy exact browser-capture evidence forward under parsed origin authority.""" - return _repair_browser_capture_origin_mismatches( - config, - raw_ids, - apply=apply, - receipt_path=receipt_path, - proof_digest=proof_digest, - ) - - -def repair_legacy_browser_capture_missing_native_ids( - config: Config, - raw_ids: list[str], - *, - apply: bool = False, - receipt_path: Path | None = None, - proof_digest: str | None = None, -) -> BrowserCaptureOriginRepairReport: - """Copy forward only the exact legacy browser shape with native_id NULL. - - The legacy raw remains immutable evidence. The parsed native identity is - written only to the new canonical raw and to the dedicated receipt. - """ - return _repair_browser_capture_origin_mismatches( - config, - raw_ids, - apply=apply, - receipt_path=receipt_path, - proof_digest=proof_digest, - allow_legacy_null_native_id=True, - receipt_schema=_LEGACY_BROWSER_CAPTURE_NATIVE_ID_REPAIR_RECEIPT_SCHEMA, - ) - - -def repair_byte_proven_browser_capture_null_native_ids( - config: Config, - raw_ids: list[str], - *, - apply: bool = False, - receipt_path: Path | None = None, - proof_digest: str | None = None, -) -> BrowserCaptureOriginRepairReport: - """Rekey only exact byte-proven browser captures with a NULL native id. - - This is deliberately narrower than the ordinary origin repair and the - quarantined legacy route: it admits only a byte-proven old unknown-key head - with a self-baseline source envelope, no retained membership census, and a - matching selected-baseline receipt. It creates a new canonical byte witness - while retaining every old source/index record unchanged. - """ - return _repair_browser_capture_origin_mismatches( - config, - raw_ids, - apply=apply, - receipt_path=receipt_path, - proof_digest=proof_digest, - allow_byte_proven_null_native_id_rekey=True, - receipt_schema=_BYTE_PROVEN_BROWSER_CAPTURE_REKEY_RECEIPT_SCHEMA, - ) + return tuple(_inspect_browser_capture_origin_strategy(archive_root, raw_id, conn=conn) for raw_id in raw_ids) def _browser_canonical_authority_conflict_witness( @@ -3609,7 +2896,7 @@ def ineligible(reason: str) -> BrowserCanonicalAuthorityConflictWitness: raw = conn.execute( """ - SELECT origin, source_path, blob_hash, blob_size + SELECT origin, source_path, blob_hash, blob_size, source_revision FROM source.raw_sessions WHERE raw_id = ? """, (raw_id,), @@ -3649,11 +2936,21 @@ def ineligible(reason: str) -> BrowserCanonicalAuthorityConflictWitness: head = conn.execute( """ - SELECT accepted_raw_id, accepted_source_revision, accepted_content_hash, accepted_frontier_kind + SELECT accepted_raw_id, accepted_source_revision, accepted_content_hash, + accepted_frontier_kind, accepted_frontier, decided_at_ms FROM raw_revision_heads WHERE logical_source_key = ? """, (canonical_key,), ).fetchone() + old_key = f"{Provider.UNKNOWN.value}:{session.provider_session_id}" + unknown_head = conn.execute( + """ + SELECT accepted_frontier_kind, accepted_frontier, decided_at_ms + FROM raw_revision_heads + WHERE logical_source_key = ? AND session_id = ? AND accepted_raw_id = ? + """, + (old_key, session_id, raw_id), + ).fetchone() # Deliberately not scoped to ``canonical_key``: the byte-proven-rekey # actuator's precondition rejects on *any* retained membership row for # this raw id (``COUNT(*) ... WHERE raw_id = ?`` with no key filter, @@ -3675,7 +2972,10 @@ def ineligible(reason: str) -> BrowserCanonicalAuthorityConflictWitness: competing_raw_id: str | None = None competing_content_hash: str | None = None + competing_source_revision: str | None = None competing_frontier_kind: str | None = None + competing_frontier: int | None = None + competing_decided_at_ms: int | None = None competing_decision: str | None = None competing_message_count: int | None = None divergent_message_index: int | None = None @@ -3684,7 +2984,10 @@ def ineligible(reason: str) -> BrowserCanonicalAuthorityConflictWitness: if head is not None: competing_raw_id = str(head["accepted_raw_id"]) competing_content_hash = _bytes_value(head["accepted_content_hash"]).hex() + competing_source_revision = str(head["accepted_source_revision"]) competing_frontier_kind = str(head["accepted_frontier_kind"]) + competing_frontier = int(head["accepted_frontier"]) + competing_decided_at_ms = int(head["decided_at_ms"]) application = conn.execute( """ SELECT decision FROM raw_revision_applications @@ -3752,8 +3055,16 @@ def ineligible(reason: str) -> BrowserCanonicalAuthorityConflictWitness: "session_id": session_id, "canonical_logical_source_key": canonical_key, "unknown_raw_content_hash": accepted_hash.hex(), + "unknown_source_revision": str(raw["source_revision"]), + "unknown_frontier_kind": None if unknown_head is None else str(unknown_head["accepted_frontier_kind"]), + "unknown_frontier": None if unknown_head is None else int(unknown_head["accepted_frontier"]), + "unknown_decided_at_ms": None if unknown_head is None else int(unknown_head["decided_at_ms"]), "competing_raw_id": competing_raw_id, "competing_content_hash": competing_content_hash, + "competing_source_revision": competing_source_revision, + "competing_frontier_kind": competing_frontier_kind, + "competing_frontier": competing_frontier, + "competing_decided_at_ms": competing_decided_at_ms, "competing_decision": competing_decision, "divergent_message_index": divergent_message_index, } @@ -3763,13 +3074,20 @@ def ineligible(reason: str) -> BrowserCanonicalAuthorityConflictWitness: status="ineligible", reason=base_reason, session_id=session_id, - old_logical_source_key=f"{Provider.UNKNOWN.value}:{session.provider_session_id}", + old_logical_source_key=old_key, canonical_logical_source_key=canonical_key, unknown_raw_content_hash=accepted_hash.hex(), + unknown_source_revision=str(raw["source_revision"]), + unknown_frontier_kind=None if unknown_head is None else str(unknown_head["accepted_frontier_kind"]), + unknown_frontier=None if unknown_head is None else int(unknown_head["accepted_frontier"]), + unknown_decided_at_ms=None if unknown_head is None else int(unknown_head["decided_at_ms"]), unknown_raw_message_count=len(projection.message_hashes), competing_raw_id=competing_raw_id, competing_content_hash=competing_content_hash, + competing_source_revision=competing_source_revision, competing_frontier_kind=competing_frontier_kind, + competing_frontier=competing_frontier, + competing_decided_at_ms=competing_decided_at_ms, competing_decision=competing_decision, competing_message_count=competing_message_count, divergent_message_index=divergent_message_index, @@ -3783,8 +3101,8 @@ def inspect_browser_canonical_authority_conflicts( ) -> BrowserCanonicalAuthorityConflictReport: """Build read-only evidence packets for browser-capture raws a safe rekey refuses. - Companion to :func:`repair_byte_proven_browser_capture_null_native_ids` - (polylogue-lkrc.3): re-runs that actuator's exact eligibility proof for each + Companion to :func:`inspect_browser_capture_origin_mismatches`: re-runs the + shared actuator strategy's exact eligibility proof for each raw id. A raw that comes back ``eligible``/``already_repaired`` is not a conflict -- the ordinary actuator already owns it -- and is only counted in ``resolved_count``. A raw that stays ``ineligible`` gets an enriched @@ -3864,13 +3182,7 @@ def record_browser_canonical_authority_conflict_blockers( an authoritative, context-injectable claim; an operator must explicitly judge it (mirrors ``upsert_pathology_findings_as_assertions``, #2383). - Deliberately exempt from this file's apply-flag/proof-digest/receipt - ceremony (unlike ``repair_duplicate_raw_identity`` and every other - actuator in this module): that ceremony exists because those actuators - repoint *authoritative* identity state (``raw_revision_heads``, - ``sessions``) where an error is expensive to detect and reverse, so they - need a CAS re-proof under an exclusive transaction plus a crash-durable - receipt trail. This function never touches authoritative state -- it + This function never touches authoritative identity state -- it writes exactly one ``candidate``/non-injected/private assertion, through ``upsert_assertion``'s single write chokepoint, which already refuses to resurrect a judged-terminal row (accepted/rejected/deferred/superseded) @@ -3962,8 +3274,6 @@ def record_browser_canonical_authority_conflict_blockers( # --- polylogue-t0dy: reconcile pre-#2729 duplicate-raw scheme --- -_DUPLICATE_RAW_IDENTITY_REPAIR_RECEIPT_SCHEMA = "polylogue.duplicate-raw-identity-repair.v1" - @dataclass(frozen=True, slots=True) class DuplicateRawIdentityRepairItem: @@ -3984,245 +3294,6 @@ class DuplicateRawIdentityRepairItem: repaired: bool = False -@dataclass(frozen=True, slots=True) -class DuplicateRawIdentityRepairReport: - mode: str - requested_count: int - eligible_count: int - repaired_count: int - already_repaired_count: int - ineligible_count: int - proof_digest: str - receipt_path: str | None - items: tuple[DuplicateRawIdentityRepairItem, ...] - - -def _duplicate_raw_identity_repair_targets( - items: list[DuplicateRawIdentityRepairItem], -) -> list[dict[str, object]]: - """Identify each locked target by its requested ``(stale, canonical)`` pair only. - - Deliberately narrower than the quarantined-raw receipt's full-item-proof - targets: for that actuator the accepted head is never touched by repair, - so a proven item's payload (including its head snapshot) is invariant - across "eligible" and "already_repaired" states and can safely anchor the - receipt. Here, repair *repoints* the accepted head (a fresh - ``decided_at_ms`` lands on every apply), so an item's full payload is NOT - invariant across states. Anchoring the receipt to the stable requested - identity instead lets a resumed invocation -- which necessarily re-proves - the CAS witness via a fresh ``proof_digest``, not a stale one -- validate - against the SAME on-disk receipt without a spurious "targets changed" - rejection caused only by the head's own repair-induced timestamp change. - """ - return [{"stale_raw_id": item.stale_raw_id, "canonical_raw_id": item.canonical_raw_id} for item in items] - - -@dataclass(slots=True) -class _LockedDuplicateRawIdentityRepairReceipt: - path: Path - descriptor: int - target_hash: str - terminal: bool - repair_intent_stale_raw_ids: tuple[str, ...] - torn_terminals: tuple[bytes, ...] = () - receipt_terminated: bool = True - - def close(self) -> None: - fcntl.flock(self.descriptor, fcntl.LOCK_UN) - os.close(self.descriptor) - - -def _validate_duplicate_raw_identity_repair_receipt_records( - parsed_receipt: tuple[list[dict[str, object] | bytes], bool], - *, - targets: list[dict[str, object]], - target_hash: str, -) -> tuple[bool, tuple[str, ...], tuple[bytes, ...], bool]: - """Mirror ``_validate_repair_receipt_records`` keyed on ``stale_raw_id``. - - Same planned/applied two-phase JSONL shape and torn-terminal recovery as - the quarantined-accepted-raw receipt (see that function's docstring) -- - ``repair_duplicate_raw_identity`` needs the identical crash/audit - guarantees as every other live-archive actuator in this file. - """ - records, terminated = parsed_receipt - if not records: - raise RuntimeError("existing repair receipt is empty") - planned = records[0] - if not isinstance(planned, dict): - raise RuntimeError("existing repair receipt does not start with valid planned JSON") - planned_keys = {"schema", "state", "target_hash", "targets", "repair_intent_stale_raw_ids", "planned_at_ms"} - if set(planned) != planned_keys or planned.get("schema") != _DUPLICATE_RAW_IDENTITY_REPAIR_RECEIPT_SCHEMA: - raise RuntimeError("existing repair receipt has an invalid planned record schema") - if planned.get("state") != "planned": - raise RuntimeError("existing repair receipt must start with a planned record") - if planned.get("target_hash") != target_hash or planned.get("targets") != targets: - raise RuntimeError("existing repair receipt targets do not match the proven repair set") - planned_at_ms = planned.get("planned_at_ms") - if not isinstance(planned_at_ms, int) or planned_at_ms < 0: - raise RuntimeError("existing repair receipt planned timestamp is invalid") - stale_raw_ids = tuple(str(target["stale_raw_id"]) for target in targets) - intent = planned.get("repair_intent_stale_raw_ids") - if not isinstance(intent, list) or any(not isinstance(raw_id, str) for raw_id in intent): - raise RuntimeError("existing repair receipt repair intent is invalid") - intent_ids = tuple(cast(list[str], intent)) - if len(set(intent_ids)) != len(intent_ids) or any(raw_id not in stale_raw_ids for raw_id in intent_ids): - raise RuntimeError("existing repair receipt repair intent does not match its targets") - if len(records) == 1: - if not terminated: - raise RuntimeError("existing repair receipt has a torn planned record") - return False, intent_ids, (), terminated - tail = records[1:] - applied = tail[-1] if isinstance(tail[-1], dict) else None - torn_terminals = ( - tuple(record for record in tail[:-1] if isinstance(record, bytes)) - if applied - else tuple(record for record in tail if isinstance(record, bytes)) - ) - expected_tail_length = len(torn_terminals) + (1 if applied is not None else 0) - if len(tail) != expected_tail_length or any(not fragment for fragment in torn_terminals): - raise RuntimeError("existing repair receipt has an invalid state transition") - if applied is None: - return False, intent_ids, torn_terminals, terminated - if not terminated: - raise RuntimeError("existing repair receipt has an unterminated applied record") - recovered = bool(torn_terminals) - applied_keys = { - "schema", - "state", - "target_hash", - "applied_at_ms", - "repaired_stale_raw_ids", - "proven_stale_raw_ids", - } - if recovered: - applied_keys |= {"torn_terminals"} - if set(applied) != applied_keys or applied.get("schema") != _DUPLICATE_RAW_IDENTITY_REPAIR_RECEIPT_SCHEMA: - raise RuntimeError("existing repair receipt has an invalid applied record schema") - if applied.get("state") != "applied" or applied.get("target_hash") != target_hash: - raise RuntimeError("existing repair receipt has an invalid applied target transition") - applied_at_ms = applied.get("applied_at_ms") - if not isinstance(applied_at_ms, int) or applied_at_ms < 0: - raise RuntimeError("existing repair receipt applied timestamp is invalid") - repaired_ids = applied.get("repaired_stale_raw_ids") - if ( - applied.get("proven_stale_raw_ids") != list(stale_raw_ids) - or not isinstance(repaired_ids, list) - or any(not isinstance(raw_id, str) for raw_id in repaired_ids) - or len(set(cast(list[str], repaired_ids))) != len(repaired_ids) - or any(raw_id not in intent_ids for raw_id in cast(list[str], repaired_ids)) - ): - raise RuntimeError("existing repair receipt applied ids do not match the planned targets") - expected_torn_witnesses = [ - {"bytes": len(fragment), "sha256": hashlib.sha256(fragment).hexdigest()} for fragment in torn_terminals - ] - if recovered and applied.get("torn_terminals") != expected_torn_witnesses: - raise RuntimeError("existing repair receipt recovery does not match its preserved torn terminal") - return True, intent_ids, torn_terminals, terminated - - -def _lock_duplicate_raw_identity_repair_receipt( - path: Path, - items: list[DuplicateRawIdentityRepairItem], -) -> _LockedDuplicateRawIdentityRepairReceipt: - """Lock one stable receipt inode and create or validate its planned record. - - Same flock(LOCK_EX|LOCK_NB) + O_NOFOLLOW + fsync(file, then parent dir) - contract as ``_lock_quarantined_raw_repair_receipt`` -- a concurrent - ``--apply`` invocation against the same receipt path fails closed instead - of racing a bare ``Path.exists()`` check, and the planned record is - durable on disk (not merely in the process) before any mutation begins. - """ - if path.is_symlink(): - raise RuntimeError("repair receipt path must not be a symbolic link") - targets = _duplicate_raw_identity_repair_targets(items) - target_hash = hashlib.sha256(json.dumps(targets, sort_keys=True, separators=(",", ":")).encode()).hexdigest() - repair_intent_stale_raw_ids = tuple(item.stale_raw_id for item in items if item.status == "eligible") - flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0) - descriptor = os.open(path, flags, 0o600) - try: - fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError as exc: - os.close(descriptor) - raise RuntimeError("operator repair receipt is already locked by another apply") from exc - try: - opened = os.fstat(descriptor) - named = path.stat(follow_symlinks=False) - if (opened.st_dev, opened.st_ino) != (named.st_dev, named.st_ino): - raise RuntimeError("operator repair receipt path changed while it was being locked") - if opened.st_size: - terminal, existing_intent, torn_terminals, terminated = ( - _validate_duplicate_raw_identity_repair_receipt_records( - _receipt_records(descriptor), targets=targets, target_hash=target_hash - ) - ) - return _LockedDuplicateRawIdentityRepairReceipt( - path, - descriptor, - target_hash, - terminal, - existing_intent, - torn_terminals, - terminated, - ) - planned = { - "schema": _DUPLICATE_RAW_IDENTITY_REPAIR_RECEIPT_SCHEMA, - "state": "planned", - "target_hash": target_hash, - "targets": targets, - "repair_intent_stale_raw_ids": list(repair_intent_stale_raw_ids), - "planned_at_ms": int(time.time() * 1000), - } - encoded = (json.dumps(planned, sort_keys=True, separators=(",", ":")) + "\n").encode() - _write_receipt_all(descriptor, encoded) - os.fsync(descriptor) - _fsync_parent(path) - return _LockedDuplicateRawIdentityRepairReceipt( - path, descriptor, target_hash, False, repair_intent_stale_raw_ids - ) - except Exception: - fcntl.flock(descriptor, fcntl.LOCK_UN) - os.close(descriptor) - raise - - -def _finish_duplicate_raw_identity_repair_receipt( - receipt: _LockedDuplicateRawIdentityRepairReceipt, - *, - items: list[DuplicateRawIdentityRepairItem], -) -> None: - opened = os.fstat(receipt.descriptor) - named = receipt.path.stat(follow_symlinks=False) - if (opened.st_dev, opened.st_ino) != (named.st_dev, named.st_ino): - raise RuntimeError("operator repair receipt path changed before terminal append") - os.lseek(receipt.descriptor, 0, os.SEEK_END) - preserved_torn_terminals = list(receipt.torn_terminals) - if preserved_torn_terminals and not receipt.receipt_terminated: - # Make even a complete-JSON prefix permanently distinguishable from a - # terminal record after the newline is appended, matching the - # quarantined-raw receipt's torn-write recovery contract. - _write_receipt_all(receipt.descriptor, b"\xff\n") - preserved_torn_terminals[-1] += b"\xff" - terminal: dict[str, object] = { - "schema": _DUPLICATE_RAW_IDENTITY_REPAIR_RECEIPT_SCHEMA, - "state": "applied", - "target_hash": receipt.target_hash, - "applied_at_ms": int(time.time() * 1000), - "repaired_stale_raw_ids": [item.stale_raw_id for item in items if item.repaired], - "proven_stale_raw_ids": [item.stale_raw_id for item in items], - } - if preserved_torn_terminals: - terminal["torn_terminals"] = [ - {"bytes": len(fragment), "sha256": hashlib.sha256(fragment).hexdigest()} - for fragment in preserved_torn_terminals - ] - _write_receipt_all( - receipt.descriptor, (json.dumps(terminal, sort_keys=True, separators=(",", ":")) + "\n").encode() - ) - os.fsync(receipt.descriptor) - _fsync_parent(receipt.path) - - def _duplicate_raw_identity_ineligible( stale_raw_id: str, canonical_raw_id: str, reason: str ) -> DuplicateRawIdentityRepairItem: @@ -4444,128 +3515,6 @@ def _apply_duplicate_raw_identity_repair(conn: sqlite3.Connection, item: Duplica ) -def repair_duplicate_raw_identity( - config: Config, - pairs: list[tuple[str, str]], - *, - apply: bool = False, - receipt_path: Path | None = None, - proof_digest: str | None = None, -) -> DuplicateRawIdentityRepairReport: - """Repoint an accepted head from a pre-#2729 duplicate raw to its post-fix twin. - - polylogue-t0dy: PR #2729 aligned the one-shot importer and the live daemon - watcher on one deterministic raw-id scheme (no ``native_id``) so *new* - ingests of a grouped/split-session file converge on one raw row instead of - duplicating. It explicitly does not retroactively repair raw pairs that - already duplicated under the OLD, native_id-inclusive scheme before that - fix landed: the accepted head stays bound to the stale raw, and its - post-fix, correctly-keyed twin sits orphaned, so every later daemon - catch-up pass over that file hits ``RuntimeError: membership replay cannot - retire an unrelated accepted head`` (archive.py). - - Each ``(stale_raw_id, canonical_raw_id)`` pair is proven independently by - :func:`_inspect_duplicate_raw_identity`: both raws must be verified - byte-identical, each raw id must equal the deterministic id its own fields - (and native_id shape) predict, the stale raw must be the CURRENT accepted - head/session pointer, and the canonical raw must not already be referenced - by any head or session -- a genuinely dangling duplicate, never a - competing authority. No durable raw/blob row is ever deleted or mutated in - place; the stale raw keeps its bytes and gains an immutable ``superseded`` - application receipt. - """ - if len(pairs) != len({stale for stale, _ in pairs}) or len(pairs) != len({canonical for _, canonical in pairs}): - raise ValueError("duplicate stale or canonical raw ids are not allowed") - if not pairs or len(pairs) > _QUARANTINED_ACCEPTED_RAW_REPAIR_LIMIT: - raise ValueError("raw-id pair list must contain 1..100 entries") - for stale_raw_id, canonical_raw_id in pairs: - for raw_id in (stale_raw_id, canonical_raw_id): - if re.fullmatch(r"[0-9a-f]{64}", raw_id) is None: - raise ValueError("raw ids must be lowercase SHA-256 identifiers") - block_reason = offline_maintenance_block_reason(config, active=apply, dry_run=not apply) - if block_reason is not None: - raise RuntimeError(block_reason) - archive_root = _raw_materialization_archive_root(config) - source_db = archive_root / "source.db" - index_db = archive_root / "index.db" - if not source_db.exists() or not index_db.exists(): - raise RuntimeError("source or index tier is missing") - - def inspect(connection: sqlite3.Connection) -> list[DuplicateRawIdentityRepairItem]: - return [ - _inspect_duplicate_raw_identity(connection, archive_root, stale, canonical) for stale, canonical in pairs - ] - - with closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True)) as conn: - conn.row_factory = sqlite3.Row - conn.execute("ATTACH DATABASE ? AS source", (f"file:{source_db}?mode=ro",)) - items = inspect(conn) - aggregate = hashlib.sha256( - json.dumps([item.proof_digest for item in items], separators=(",", ":")).encode() - ).hexdigest() - if apply and any(item.status == "ineligible" for item in items): - raise RuntimeError("duplicate raw identity repair refused because one or more targets are ineligible") - if apply and receipt_path is None: - raise ValueError("apply requires an explicit operator repair receipt path") - if apply and proof_digest != aggregate: - raise RuntimeError("apply proof digest does not match the exact dry-run target list") - if apply: - from polylogue.storage.index_generation import RebuildLease - - assert receipt_path is not None - receipt_path.parent.mkdir(parents=True, exist_ok=True) - with RebuildLease(archive_root): - receipt = _lock_duplicate_raw_identity_repair_receipt(receipt_path, items) - try: - with closing(sqlite3.connect(f"file:{index_db}?mode=rw", uri=True)) as conn: - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA foreign_keys = ON") - conn.execute("ATTACH DATABASE ? AS source", (f"file:{source_db}?mode=ro",)) - conn.execute("BEGIN IMMEDIATE") - try: - locked = inspect(conn) - locked_digest = hashlib.sha256( - json.dumps([item.proof_digest for item in locked], separators=(",", ":")).encode() - ).hexdigest() - if locked_digest != proof_digest: - raise RuntimeError("authority proof changed after acquiring the repair transaction") - if any(item.status == "ineligible" for item in locked): - raise RuntimeError("a duplicate raw identity target became ineligible") - if receipt.terminal and any(item.status != "already_repaired" for item in locked): - raise RuntimeError("terminal operator receipt disagrees with durable index authority") - if receipt.torn_terminals and any(item.status != "already_repaired" for item in locked): - raise RuntimeError("torn terminal receipt has no matching committed index refinement") - for item in locked: - if item.status == "eligible": - _apply_duplicate_raw_identity_repair(conn, item) - after = inspect(conn) - if any(item.status != "already_repaired" for item in after): - raise RuntimeError("duplicate raw identity repair did not reach terminal state") - conn.commit() - except Exception: - conn.rollback() - raise - items = [ - dataclasses.replace(after_item, repaired=before.status == "eligible") - for before, after_item in zip(locked, after, strict=True) - ] - if not receipt.terminal: - _finish_duplicate_raw_identity_repair_receipt(receipt, items=items) - finally: - receipt.close() - return DuplicateRawIdentityRepairReport( - mode="apply" if apply else "dry-run", - requested_count=len(items), - eligible_count=sum(item.status == "eligible" for item in items), - repaired_count=sum(item.repaired for item in items), - already_repaired_count=sum(item.status == "already_repaired" for item in items), - ineligible_count=sum(item.status == "ineligible" for item in items), - proof_digest=aggregate, - receipt_path=str(receipt_path) if receipt_path is not None else None, - items=tuple(items), - ) - - def _format_bytes(value: int) -> str: units = ("B", "KiB", "MiB", "GiB", "TiB") amount = float(max(value, 0)) @@ -6526,7 +5475,9 @@ def repair_raw_materialization( interrupted=True, ) recovered_census_count = len(recovered_censuses) - blocker_count = unresolved_raw_authority_blockers(archive_root) + from polylogue.storage.raw_authority import unresolved_raw_replay_blockers + + blocker_count = unresolved_raw_replay_blockers(archive_root) if blocker_count: return _internal_derived_repair_result( "raw_materialization", diff --git a/tests/unit/cli/test_archive_maintenance_cli.py b/tests/unit/cli/test_archive_maintenance_cli.py index 3ecf65a687..fc933ffb95 100644 --- a/tests/unit/cli/test_archive_maintenance_cli.py +++ b/tests/unit/cli/test_archive_maintenance_cli.py @@ -5,6 +5,7 @@ import os import sqlite3 from pathlib import Path +from unittest.mock import patch import pytest from click.testing import CliRunner @@ -15,7 +16,6 @@ from polylogue.core.enums import Provider from polylogue.core.json import json_document from polylogue.maintenance.replay import rebuild_index_from_source -from polylogue.sources.live.cursor import CursorStore from polylogue.storage.blob_gc import read_gc_history from polylogue.storage.blob_publication import ArchiveBlobPublisher from polylogue.storage.raw_authority import RawReplayPlan, record_raw_authority_census @@ -145,7 +145,16 @@ def test_raw_authority_blocker_resolution_cli_requires_confirmation( ) -> None: calls: list[tuple[str, str]] = [] - def resolve(_root: Path, blocker_id: str, *, resolution: str) -> dict[str, object]: + def resolve( + _root: Path, + blocker_id: str, + *, + resolution: str, + assertion_id: str | None = None, + judgment_disposition: str | None = None, + ) -> dict[str, object]: + assert assertion_id is None + assert judgment_disposition is None calls.append((blocker_id, resolution)) return {"blocker_id": blocker_id, "current_plan": {"plan_id": "current-plan"}} @@ -226,36 +235,6 @@ def _seed_assertion_export_rows(archive_root: Path) -> None: ) -def _seed_missing_blob_cursor(archive_root: Path, source: Path) -> None: - source.parent.mkdir(parents=True, exist_ok=True) - source.write_text('{"type":"session_meta","payload":{"id":"missing-blob"}}\n', encoding="utf-8") - blob_hash = b"a" * 32 - with sqlite3.connect(archive_root / "source.db") as conn: - write_source_raw_session_blob_ref( - conn, - origin="codex-session", - source_path=str(source), - source_index=0, - blob_hash=blob_hash, - blob_size=source.stat().st_size, - acquired_at_ms=1, - native_id="missing-blob", - ) - stat = source.stat() - CursorStore(archive_root / "ops.db").set( - source, - stat.st_size, - byte_offset=stat.st_size, - last_complete_newline=stat.st_size, - parser_fingerprint="live-batched-v2", - content_fingerprint=blob_hash.hex(), - source_name="codex", - st_dev=stat.st_dev, - st_ino=stat.st_ino, - mtime_ns=stat.st_mtime_ns, - ) - - def _create_user_v3(path: Path) -> None: path.unlink(missing_ok=True) with sqlite3.connect(path) as conn: @@ -1436,51 +1415,17 @@ def test_archive_maintenance_help_omits_copy_activation_surface(cli_runner: CliR assert removed not in result.output -def test_missing_raw_blob_cursor_repair_dry_run_keeps_cursor( - cli_workspace: dict[str, Path], - cli_runner: CliRunner, -) -> None: - source = cli_workspace["archive_root"] / "watch" / "missing-blob.jsonl" - _seed_missing_blob_cursor(cli_workspace["archive_root"], source) - - result = cli_runner.invoke( - cli, - [ - "--plain", - "ops", - "maintenance", - "missing-raw-blob-cursors", - "--output-format", - "json", - ], - catch_exceptions=False, - ) - - assert result.exit_code == 0 - payload = json.loads(result.output) - assert payload["mode"] == "dry-run" - assert payload["candidate_count"] == 1 - assert payload["deleted_cursor_count"] == 0 - assert payload["candidates"][0]["source_path"] == str(source) - with sqlite3.connect(cli_workspace["archive_root"] / "ops.db") as conn: - assert conn.execute("SELECT 1 FROM ingest_cursor WHERE source_path = ?", (str(source),)).fetchone() == (1,) - - -def test_missing_raw_blob_cursor_repair_apply_deletes_only_cursor( +def test_raw_authority_frontier_cli_replaces_incident_specific_commands( cli_workspace: dict[str, Path], cli_runner: CliRunner, ) -> None: - source = cli_workspace["archive_root"] / "watch" / "missing-blob.jsonl" - _seed_missing_blob_cursor(cli_workspace["archive_root"], source) - result = cli_runner.invoke( cli, [ "--plain", "ops", "maintenance", - "missing-raw-blob-cursors", - "--apply", + "raw-authority-frontier", "--output-format", "json", ], @@ -1489,331 +1434,55 @@ def test_missing_raw_blob_cursor_repair_apply_deletes_only_cursor( assert result.exit_code == 0 payload = json.loads(result.output) - assert payload["mode"] == "apply" - assert payload["candidate_count"] == 1 - assert payload["deleted_cursor_count"] == 1 - with sqlite3.connect(cli_workspace["archive_root"] / "ops.db") as conn: - assert conn.execute("SELECT 1 FROM ingest_cursor WHERE source_path = ?", (str(source),)).fetchone() is None - with sqlite3.connect(cli_workspace["archive_root"] / "source.db") as conn: - assert conn.execute("SELECT 1 FROM raw_sessions WHERE source_path = ?", (str(source),)).fetchone() == (1,) - - -def test_quarantined_accepted_raw_repair_cli_dry_run_is_bounded_json( - cli_workspace: dict[str, Path], - cli_runner: CliRunner, -) -> None: - del cli_workspace - raw_id = "a" * 64 - result = cli_runner.invoke( - cli, - [ - "--plain", - "ops", - "maintenance", - "quarantined-accepted-raws", - "--raw-id", - raw_id, - "--output-format", - "json", - ], - catch_exceptions=False, - ) - assert result.exit_code == 0 - payload = json.loads(result.output) - assert payload["mode"] == "dry-run" - assert payload["requested_count"] == 1 - assert payload["ineligible_count"] == 1 - assert len(payload["proof_digest"]) == 64 - assert payload["items"][0]["raw_id"] == raw_id - - plain = cli_runner.invoke( - cli, - ["--plain", "ops", "maintenance", "quarantined-accepted-raws", "--raw-id", raw_id], - catch_exceptions=False, - ) - assert plain.exit_code == 0 - assert f"Proof digest: {payload['proof_digest']}" in plain.output - assert f"{raw_id} ineligible proof=unavailable" in plain.output - - -def test_quarantined_accepted_raw_repair_cli_apply_requires_receipt_and_proof( - cli_workspace: dict[str, Path], - cli_runner: CliRunner, -) -> None: - del cli_workspace - raw_id = "a" * 64 - missing_receipt = cli_runner.invoke( - cli, - ["--plain", "ops", "maintenance", "quarantined-accepted-raws", "--raw-id", raw_id, "--apply"], - ) - assert missing_receipt.exit_code == 2 - assert "--receipt" in missing_receipt.output - missing_proof = cli_runner.invoke( - cli, - [ - "--plain", - "ops", - "maintenance", - "quarantined-accepted-raws", - "--raw-id", - raw_id, - "--apply", - "--receipt", - "repair.jsonl", - ], - ) - assert missing_proof.exit_code == 2 - assert "--proof-digest" in missing_proof.output - - -def test_browser_capture_origin_repair_cli_is_bounded_and_requires_receipt_proof( - cli_workspace: dict[str, Path], - cli_runner: CliRunner, -) -> None: - del cli_workspace - raw_id = "b" * 64 - dry_run = cli_runner.invoke( - cli, - [ - "--plain", - "ops", - "maintenance", - "browser-capture-origin-mismatches", - "--raw-id", - raw_id, - "--output-format", - "json", - ], - catch_exceptions=False, - ) - assert dry_run.exit_code == 0 - payload = json.loads(dry_run.output) - assert payload["mode"] == "dry-run" - assert payload["requested_count"] == 1 - assert payload["ineligible_count"] == 1 - assert len(payload["proof_digest"]) == 64 - - missing_receipt = cli_runner.invoke( - cli, - [ - "--plain", - "ops", - "maintenance", - "browser-capture-origin-mismatches", - "--raw-id", - raw_id, - "--apply", - ], - ) - assert missing_receipt.exit_code == 2 - assert "--receipt" in missing_receipt.output - missing_proof = cli_runner.invoke( - cli, - [ - "--plain", - "ops", - "maintenance", - "browser-capture-origin-mismatches", - "--raw-id", - raw_id, - "--apply", - "--receipt", - "repair.jsonl", - ], - ) - assert missing_proof.exit_code == 2 - assert "--proof-digest" in missing_proof.output - - -def test_legacy_browser_native_id_repair_cli_is_bounded_and_requires_receipt_proof( - cli_workspace: dict[str, Path], - cli_runner: CliRunner, -) -> None: - del cli_workspace - raw_id = "c" * 64 - dry_run = cli_runner.invoke( - cli, - [ - "--plain", - "ops", - "maintenance", - "legacy-browser-capture-missing-native-id", - "--raw-id", - raw_id, - "--output-format", - "json", - ], - catch_exceptions=False, - ) - assert dry_run.exit_code == 0 - assert json.loads(dry_run.output)["ineligible_count"] == 1 - missing_receipt = cli_runner.invoke( - cli, - [ - "--plain", - "ops", - "maintenance", - "legacy-browser-capture-missing-native-id", - "--raw-id", - raw_id, - "--apply", - ], - ) - assert missing_receipt.exit_code == 2 - assert "--receipt" in missing_receipt.output - - -def test_browser_canonical_authority_conflicts_cli_is_read_only_by_default( - cli_workspace: dict[str, Path], - cli_runner: CliRunner, -) -> None: - del cli_workspace - raw_id = "d" * 64 - dry_run = cli_runner.invoke( - cli, - [ - "--plain", - "ops", - "maintenance", - "browser-canonical-authority-conflicts", - "--raw-id", - raw_id, - "--output-format", - "json", - ], - catch_exceptions=False, - ) - assert dry_run.exit_code == 0 - payload = json.loads(dry_run.output) - assert payload["requested_count"] == 1 - assert "assertion_ids" not in payload - - plain = cli_runner.invoke( - cli, - ["--plain", "ops", "maintenance", "browser-canonical-authority-conflicts", "--raw-id", raw_id], - catch_exceptions=False, - ) - assert plain.exit_code == 0 - assert "Blocker:" not in plain.output + assert payload["accepted_head_count"] == 0 + assert payload["plan_count"] == 0 + assert payload["executable_plan_count"] == 0 + assert payload["state_counts"] == {} + assert payload["query_handle"].startswith("polylogue://raw-authority-census/") + help_result = cli_runner.invoke(cli, ["--plain", "ops", "maintenance", "--help"]) + assert help_result.exit_code == 0 + assert "raw-authority-frontier" in help_result.output + for removed in ( + "missing-raw-blob-cursors", + "quarantined-accepted-raws", + "browser-capture-origin-mismatches", + "legacy-browser-capture-missing-native-id", + "browser-canonical-authority-conflicts", + "duplicate-raw-identity", + ): + assert removed not in help_result.output -def test_browser_canonical_authority_conflicts_cli_record_calls_the_recording_path( - cli_workspace: dict[str, Path], - cli_runner: CliRunner, -) -> None: - """``--record`` routes through ``record_browser_canonical_authority_conflict_blockers``. - - Exercises the CLI adapter's ``--record`` branch (polylogue-hleq NIT): the - JSON payload gains an ``assertion_ids`` key (absent without ``--record``, - per the sibling read-only test) and the plain-output loop over - ``assertion_ids`` runs without error. A raw id with no resolvable session - (this test's fixture) is exactly the shape - ``record_browser_canonical_authority_conflict_blockers`` itself declines - to persist a blocker for (no ``session_id`` to target), so this proves the - CLI calls the recording function and surfaces its real (empty) result - rather than fabricating one; the deep persistence/idempotency/judged-row- - protection behavior of the recording function itself is covered at the - storage layer in ``test_browser_capture_origin_repair.py``. - """ - archive_root = cli_workspace["archive_root"] - raw_id = "e" * 64 - dry_run = cli_runner.invoke( + apply_without_confirmation = cli_runner.invoke( cli, [ "--plain", "ops", "maintenance", - "browser-canonical-authority-conflicts", - "--raw-id", - raw_id, - "--record", - "--output-format", - "json", + "raw-authority-frontier", + "--apply-plan", + "raw-authority-frontier:" + "a" * 64, + "--preview-census", + payload["census_id"], ], - catch_exceptions=False, - ) - assert dry_run.exit_code == 0 - payload = json.loads(dry_run.output) - assert payload["assertion_ids"] == [] - with sqlite3.connect(archive_root / "user.db") as user_conn: - count = user_conn.execute("SELECT COUNT(*) FROM assertions WHERE kind = 'blocker'").fetchone()[0] - assert count == 0 - - plain = cli_runner.invoke( - cli, - ["--plain", "ops", "maintenance", "browser-canonical-authority-conflicts", "--raw-id", raw_id, "--record"], - catch_exceptions=False, ) - assert plain.exit_code == 0 - assert "Blocker:" not in plain.output + assert apply_without_confirmation.exit_code == 1 + assert "without --yes" in apply_without_confirmation.output -def test_duplicate_raw_identity_cli_dry_run_is_bounded_and_requires_receipt_proof( +def test_raw_authority_frontier_cli_refuses_durable_census_while_daemon_runs( cli_workspace: dict[str, Path], cli_runner: CliRunner, ) -> None: - del cli_workspace - stale_raw_id = "f" * 64 - canonical_raw_id = "0" * 64 - dry_run = cli_runner.invoke( - cli, - [ - "--plain", - "ops", - "maintenance", - "duplicate-raw-identity", - "--pair", - f"{stale_raw_id}:{canonical_raw_id}", - "--output-format", - "json", - ], - catch_exceptions=False, - ) - assert dry_run.exit_code == 0 - payload = json.loads(dry_run.output) - assert payload["mode"] == "dry-run" - assert payload["requested_count"] == 1 - assert payload["ineligible_count"] == 1 - assert payload["items"][0]["stale_raw_id"] == stale_raw_id - assert payload["items"][0]["canonical_raw_id"] == canonical_raw_id - - malformed = cli_runner.invoke( - cli, - ["--plain", "ops", "maintenance", "duplicate-raw-identity", "--pair", "not-a-pair"], - ) - assert malformed.exit_code == 2 - assert "STALE_RAW_ID:CANONICAL_RAW_ID" in malformed.output + """A census reconciles durable obligations, so it needs writer exclusion.""" + with patch("polylogue.maintenance.offline_guard.running_daemon_pid", return_value=123): + result = cli_runner.invoke( + cli, + ["--plain", "ops", "maintenance", "raw-authority-frontier", "--output-format", "json"], + ) - missing_receipt = cli_runner.invoke( - cli, - [ - "--plain", - "ops", - "maintenance", - "duplicate-raw-identity", - "--pair", - f"{stale_raw_id}:{canonical_raw_id}", - "--apply", - ], - ) - assert missing_receipt.exit_code == 2 - assert "--receipt" in missing_receipt.output - missing_proof = cli_runner.invoke( - cli, - [ - "--plain", - "ops", - "maintenance", - "duplicate-raw-identity", - "--pair", - f"{stale_raw_id}:{canonical_raw_id}", - "--apply", - "--receipt", - "repair.jsonl", - ], - ) - assert missing_proof.exit_code == 2 - assert "--proof-digest" in missing_proof.output + assert result.exit_code == 1 + assert "Refusing offline maintenance while polylogued PID 123 is running" in result.output def test_archive_read_cli_lists_archive_sessions( diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index 5f9944ab51..c1da613c17 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -144,6 +144,8 @@ def test_polylogued_status_plain_reports_daemon_components(tmp_path: Path) -> No def test_polylogued_status_json_reports_archive_storage(tmp_path: Path) -> None: + from polylogue.storage.raw_reconciler import inspect_raw_authority_frontier + for filename, tier in ( ("source.db", ArchiveTier.SOURCE), ("index.db", ArchiveTier.INDEX), @@ -154,6 +156,9 @@ def test_polylogued_status_json_reports_archive_storage(tmp_path: Path) -> None: with sqlite3.connect(tmp_path / "embeddings.db") as conn: conn.execute(f"PRAGMA user_version = {EMBEDDINGS_SCHEMA_VERSION}") conn.commit() + inspect_raw_authority_frontier( + Config(archive_root=tmp_path, render_root=tmp_path / "render", sources=[], db_path=tmp_path / "index.db") + ) with ( patch("polylogue.daemon.status.archive_root", return_value=tmp_path), @@ -546,6 +551,17 @@ def fake_repair_raw_materialization(config: Config, *, dry_run: bool, raw_artifa calls["raw_artifact_limit"] = raw_artifact_limit return FakeResult() + def fake_recover(config: Config) -> tuple[str, ...]: + order.append("recover-frontier") + calls["recover_archive_root"] = config.archive_root + return () + + def fake_converge(config: Config, *, limit: int) -> int: + order.append("frontier") + calls["frontier_archive_root"] = config.archive_root + calls["frontier_limit"] = limit + return 3 + monkeypatch.setattr("polylogue.paths.archive_root", lambda: tmp_path / "archive") monkeypatch.setattr("polylogue.paths.render_root", lambda: tmp_path / "render") monkeypatch.setattr( @@ -553,9 +569,14 @@ def fake_repair_raw_materialization(config: Config, *, dry_run: bool, raw_artifa fake_restore_direct_blob_reference_debt, ) monkeypatch.setattr("polylogue.storage.repair.repair_raw_materialization", fake_repair_raw_materialization) + monkeypatch.setattr( + "polylogue.storage.raw_reconciler.recover_interrupted_raw_authority_frontier", + fake_recover, + ) + monkeypatch.setattr(daemon_cli, "_converge_raw_authority_frontier", fake_converge) - assert daemon_cli._drain_raw_materialization_once(limit=11) == 7 - assert order == ["restore", "materialize"] + assert daemon_cli._drain_raw_materialization_once(limit=11) == 10 + assert order == ["restore", "recover-frontier", "materialize", "frontier"] assert calls == { "restore_db_path": tmp_path / "archive" / "source.db", "restore_dry_run": False, @@ -565,6 +586,9 @@ def fake_repair_raw_materialization(config: Config, *, dry_run: bool, raw_artifa "render_root": tmp_path / "render", "dry_run": False, "raw_artifact_limit": 11, + "recover_archive_root": tmp_path / "archive", + "frontier_archive_root": tmp_path / "archive", + "frontier_limit": 8, } diff --git a/tests/unit/storage/test_archive_readiness.py b/tests/unit/storage/test_archive_readiness.py index 892a22980c..9bc379f5c5 100644 --- a/tests/unit/storage/test_archive_readiness.py +++ b/tests/unit/storage/test_archive_readiness.py @@ -7,12 +7,87 @@ from polylogue.archive.revision_authority import BYTE_AUTHORITY_CENSUS_DETAIL from polylogue.storage.archive_readiness import raw_materialization_readiness_snapshot, raw_materialization_ready +from polylogue.storage.raw_authority import ( + RawReplayPlan, + RawReplayPlanOutcome, + RawReplayPlanStatus, + finalize_raw_authority_census, + record_raw_authority_census, + record_raw_replay_outcome, +) +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root def _category_counts(snapshot: Mapping[str, object]) -> Mapping[str, object]: return cast(Mapping[str, object], snapshot["category_counts"]) +def test_raw_materialization_readiness_requires_completed_frontier_census() -> None: + counters_green: dict[str, object] = {"available": True} + + assert raw_materialization_ready(counters_green) is False + assert ( + raw_materialization_ready({**counters_green, "raw_authority_frontier": {"lifecycle_status": "interrupted"}}) + is False + ) + assert ( + raw_materialization_ready({**counters_green, "raw_authority_frontier": {"lifecycle_status": "completed"}}) + is True + ) + + +def test_readiness_uses_frontier_postflight_not_preapply_scope(tmp_path: Path) -> None: + """An applied repair must not remain blocked by its immutable preflight.""" + initialize_active_archive_root(tmp_path) + plan = RawReplayPlan( + plan_id="raw-authority-frontier:" + "a" * 64, + input_digest="b" * 64, + input_raw_ids=("raw-1",), + logical_keys=("chatgpt-export:conversation-1",), + authority_witness={"schema": "polylogue.raw-authority-frontier-plan.v1"}, + source_preconditions={}, + index_preconditions={}, + ) + receipt = record_raw_authority_census( + tmp_path, + (plan,), + selected_plan_ids={plan.plan_id}, + executable_plan_ids={plan.plan_id}, + mode="apply", + quiescent=True, + scope={ + "schema": "polylogue.raw-authority-frontier-scope.v1", + "state_counts": {"missing_source_bytes": 1}, + }, + residual={ + "schema": "polylogue.raw-authority-frontier-residual.v1", + "state_counts": {"missing_source_bytes": 1}, + }, + ) + record_raw_replay_outcome( + tmp_path, + receipt.census_id, + RawReplayPlanOutcome( + plan_id=plan.plan_id, + input_raw_ids=plan.input_raw_ids, + status=RawReplayPlanStatus.EXECUTED, + reason="fixture repaired the exact plan", + next_action="none", + ), + ) + finalize_raw_authority_census( + tmp_path, + receipt.census_id, + post_plans=(), + post_residual={"schema": "polylogue.raw-authority-frontier-residual.v1", "state_counts": {}}, + ) + + snapshot = raw_materialization_readiness_snapshot(tmp_path) + + assert snapshot["raw_authority_frontier_blocking_count"] == 0 + assert raw_materialization_ready(snapshot) is True + + def test_raw_materialization_snapshot_classifies_durable_authority_gaps(tmp_path: Path) -> None: source_db = tmp_path / "source.db" index_db = tmp_path / "index.db" @@ -773,6 +848,7 @@ def test_raw_materialization_ready_rejects_failed_debt_classifier() -> None: """ clean = { "available": True, + "raw_authority_frontier": {"lifecycle_status": "completed"}, "critical": 0, "warning": 0, "actionable": 0, diff --git a/tests/unit/storage/test_browser_capture_origin_repair.py b/tests/unit/storage/test_browser_capture_origin_repair.py index e46c36436e..a5247fa985 100644 --- a/tests/unit/storage/test_browser_capture_origin_repair.py +++ b/tests/unit/storage/test_browser_capture_origin_repair.py @@ -1,33 +1,29 @@ from __future__ import annotations -import hashlib import json import sqlite3 -import subprocess -import sys from contextlib import closing from pathlib import Path -from types import SimpleNamespace from typing import Any, cast import pytest from polylogue.archive.revision_replay import ApplicationDecision -from polylogue.archive.session_revision_membership import MembershipClassification from polylogue.config import Config -from polylogue.core.enums import Provider +from polylogue.core.enums import AssertionStatus, Provider from polylogue.pipeline.ids import session_content_hash, session_revision_projection -from polylogue.sources.live.batch import LiveBatchProcessor -from polylogue.sources.live.cursor import CursorStore -from polylogue.sources.revision_backfill import _parse_one, backfill_historical_revision_evidence +from polylogue.sources.revision_backfill import _parse_one from polylogue.storage.blob_store import BlobStore +from polylogue.storage.raw_authority import resolve_raw_authority_blocker +from polylogue.storage.raw_reconciler import ( + RawAuthorityActuator, + RawAuthorityFrontierState, + apply_raw_authority_frontier, + inspect_raw_authority_frontier, +) from polylogue.storage.repair import ( inspect_browser_canonical_authority_conflicts, record_browser_canonical_authority_conflict_blockers, - repair_browser_capture_origin_mismatches, - repair_byte_proven_browser_capture_null_native_ids, - repair_legacy_browser_capture_missing_native_ids, - repair_quarantined_accepted_raws, ) from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root @@ -35,6 +31,7 @@ RevisionApplicationReceipt, record_revision_application_sync, ) +from polylogue.storage.sqlite.archive_tiers.user_write import mark_assertion_status def _config(root: Path) -> Config: @@ -585,1625 +582,80 @@ def _journal_modes(root: Path) -> dict[str, str]: return modes -def test_browser_capture_origin_copy_forward_preserves_old_evidence_and_is_idempotent(tmp_path: Path) -> None: +def test_unified_frontier_applies_browser_origin_without_incident_receipt(tmp_path: Path) -> None: raw_id = _seed_mismatched_browser_head(tmp_path) - old_evidence = { - "raw": _rows(tmp_path, "source", "raw_sessions", "raw_id = ?", (raw_id,)), - "blob": _rows(tmp_path, "source", "blob_refs", "ref_id = ?", (raw_id,)), - "membership": _rows(tmp_path, "source", "raw_session_memberships", "raw_id = ?", (raw_id,)), - "census": _rows(tmp_path, "source", "raw_membership_census", "raw_id = ?", (raw_id,)), - "head": _rows(tmp_path, "index", "raw_revision_heads", "accepted_raw_id = ?", (raw_id,)), - "application": _rows(tmp_path, "index", "raw_revision_applications", "raw_id = ?", (raw_id,)), - } - - dry_run = repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]) - - assert dry_run.eligible_count == 1, dry_run.items[0].reason - item = dry_run.items[0] - assert item.canonical_origin == "chatgpt-export" - assert item.canonical_logical_source_key == "chatgpt:browser-origin-one" - assert item.copy_forward_raw_id not in {None, raw_id} - assert repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]).ineligible_count == 1 - - receipt = tmp_path / "recovery" / "browser-origin.jsonl" - applied = repair_browser_capture_origin_mismatches( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - - assert applied.repaired_count == 1 - assert applied.items[0].status == "already_repaired" - assert [json.loads(line)["state"] for line in receipt.read_text().splitlines()] == ["planned", "applied"] - for name, before in old_evidence.items(): - tier = "index" if name in {"head", "application"} else "source" - table = { - "raw": "raw_sessions", - "blob": "blob_refs", - "membership": "raw_session_memberships", - "census": "raw_membership_census", - "head": "raw_revision_heads", - "application": "raw_revision_applications", - }[name] - key = "accepted_raw_id" if name == "head" else "raw_id" if name not in {"blob"} else "ref_id" - assert _rows(tmp_path, tier, table, f"{key} = ?", (raw_id,)) == before - copy_raw_id = applied.items[0].copy_forward_raw_id - assert copy_raw_id is not None - with sqlite3.connect(tmp_path / "source.db") as source: - assert source.execute( - "SELECT origin, logical_source_key, revision_authority FROM raw_sessions WHERE raw_id = ?", - (copy_raw_id,), - ).fetchone() == ("chatgpt-export", "chatgpt:browser-origin-one", "byte_proven") - with sqlite3.connect(tmp_path / "index.db") as index: - index.execute("ATTACH DATABASE ? AS source", (str(tmp_path / "source.db"),)) - assert index.execute( - "SELECT raw_id FROM sessions WHERE session_id = 'chatgpt-export:browser-origin-one'" - ).fetchone() == (copy_raw_id,) - assert index.execute( - "SELECT accepted_raw_id FROM raw_revision_heads WHERE logical_source_key = 'chatgpt:browser-origin-one'" - ).fetchone() == (copy_raw_id,) - assert index.execute( - """ - SELECT COUNT(*) FROM raw_revision_heads AS h - JOIN sessions AS s ON s.session_id = h.session_id AND s.raw_id = h.accepted_raw_id - JOIN source.raw_sessions AS r ON r.raw_id = h.accepted_raw_id - WHERE r.origin = 'unknown-export' - """ - ).fetchone() == (0,) - reapplied = repair_browser_capture_origin_mismatches( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - assert reapplied.repaired_count == 0 - assert receipt.read_text().count("\n") == 2 - - -def test_browser_origin_repair_resumes_prelegacy_planned_receipt(tmp_path: Path) -> None: - import polylogue.storage.repair as repair_module - - raw_id = _seed_mismatched_browser_head(tmp_path) - dry_run = repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]) - historical_target = repair_module._browser_origin_item_payload(dry_run.items[0]) - # Emulate the exact v1 ordinary target shape written before the legacy - # actuator added its provenance-only fields. - historical_target.pop("legacy_null_native_id", None) - historical_target.pop("parser_derived_native_id", None) - target_hash = hashlib.sha256( - json.dumps([historical_target], sort_keys=True, separators=(",", ":")).encode() - ).hexdigest() - receipt = tmp_path / "prelegacy-planned.jsonl" - receipt.write_text( - json.dumps( - { - "schema": "polylogue.browser-capture-origin-copy-forward.v1", - "state": "planned", - "target_hash": target_hash, - "targets": [historical_target], - "planned_at_ms": 1, - }, - sort_keys=True, - separators=(",", ":"), - ) - + "\n" - ) - - applied = repair_browser_capture_origin_mismatches( - _config(tmp_path), [raw_id], apply=True, receipt_path=receipt, proof_digest=dry_run.proof_digest - ) - - assert applied.repaired_count == 1 - assert [json.loads(line)["state"] for line in receipt.read_text().splitlines()] == ["planned", "applied"] - - -def test_browser_capture_origin_rejects_decided_quarantined_membership(tmp_path: Path) -> None: - raw_id = _seed_mismatched_browser_head(tmp_path) - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute( - """ - UPDATE raw_session_memberships - SET decision = 'ambiguous', decided_at_ms = 2 - WHERE raw_id = ? - """, - (raw_id,), - ) - - report = repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]) - - assert report.ineligible_count == 1 - assert report.items[0].reason == "membership census does not exactly reproduce the accepted session" - - -def test_browser_capture_origin_rebuild_keeps_copy_forward_head(tmp_path: Path) -> None: - raw_id = _seed_mismatched_browser_head(tmp_path) - dry_run = repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]) - applied = repair_browser_capture_origin_mismatches( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=tmp_path / "rebuild-repair.jsonl", - proof_digest=dry_run.proof_digest, - ) - copy_raw_id = applied.items[0].copy_forward_raw_id - assert copy_raw_id is not None - - backfill_historical_revision_evidence(tmp_path, selected_raw_ids=[raw_id, copy_raw_id]) - - with sqlite3.connect(tmp_path / "index.db") as index: - assert index.execute( - "SELECT accepted_raw_id FROM raw_revision_heads WHERE logical_source_key = 'chatgpt:browser-origin-one'" - ).fetchone() == (copy_raw_id,) - - -@pytest.mark.parametrize( - "mutation", - [ - "blob", - "head", - "origin", - "application", - "generation", - "authority", - "native_id", - "source_index", - "blob_ref_path", - "predecessor_source", - "predecessor_raw", - "append_start", - "append_end", - "capture_mode", - "canonical_head", - ], -) -def test_browser_capture_origin_copy_forward_mutations_fail_closed(tmp_path: Path, mutation: str) -> None: - raw_id = _seed_mismatched_browser_head(tmp_path) - with ( - closing(sqlite3.connect(tmp_path / "source.db")) as source, - closing(sqlite3.connect(tmp_path / "index.db")) as index, - ): - if mutation == "blob": - source.execute("UPDATE blob_refs SET size_bytes = size_bytes + 1 WHERE ref_id = ?", (raw_id,)) - elif mutation == "head": - index.execute("UPDATE raw_revision_heads SET accepted_frontier = accepted_frontier + 1") - elif mutation == "origin": - source.execute("UPDATE raw_sessions SET origin = 'chatgpt-export' WHERE raw_id = ?", (raw_id,)) - elif mutation == "application": - index.execute( - "DELETE FROM raw_revision_applications WHERE raw_id = ? AND decision = 'selected_baseline'", - (raw_id,), - ) - elif mutation == "generation": - source.execute("UPDATE raw_sessions SET acquisition_generation = 1 WHERE raw_id = ?", (raw_id,)) - source.execute("UPDATE raw_session_memberships SET acquisition_generation = 1 WHERE raw_id = ?", (raw_id,)) - index.execute( - "UPDATE raw_revision_heads SET acquisition_generation = 1 WHERE accepted_raw_id = ?", (raw_id,) - ) - index.execute( - "UPDATE raw_revision_applications SET acquisition_generation = 1 WHERE raw_id = ?", - (raw_id,), - ) - elif mutation == "authority": - source.execute("UPDATE raw_sessions SET revision_authority = 'byte_proven' WHERE raw_id = ?", (raw_id,)) - elif mutation == "native_id": - source.execute("UPDATE raw_sessions SET native_id = 'wrong-native-id' WHERE raw_id = ?", (raw_id,)) - elif mutation == "source_index": - source.execute("UPDATE raw_sessions SET source_index = 1 WHERE raw_id = ?", (raw_id,)) - elif mutation == "blob_ref_path": - source.execute( - "UPDATE blob_refs SET source_path = 'browser-capture/wrong.json' WHERE ref_id = ?", (raw_id,) - ) - elif mutation == "predecessor_source": - source.execute( - "UPDATE raw_sessions SET predecessor_source_revision = ? WHERE raw_id = ?", ("0" * 64, raw_id) - ) - elif mutation == "predecessor_raw": - source.execute("UPDATE raw_sessions SET predecessor_raw_id = ? WHERE raw_id = ?", ("f" * 64, raw_id)) - elif mutation == "append_start": - source.execute("UPDATE raw_sessions SET append_start_offset = 0 WHERE raw_id = ?", (raw_id,)) - elif mutation == "append_end": - source.execute("UPDATE raw_sessions SET append_end_offset = 1 WHERE raw_id = ?", (raw_id,)) - elif mutation == "capture_mode": - source.execute("UPDATE raw_sessions SET capture_mode = 'chatgpt' WHERE raw_id = ?", (raw_id,)) - else: - index.execute( - """ - INSERT INTO raw_revision_heads ( - logical_source_key, session_id, accepted_raw_id, accepted_source_revision, - accepted_content_hash, accepted_frontier_kind, accepted_frontier, - acquisition_generation, decided_at_ms - ) SELECT 'chatgpt:browser-origin-one', session_id, accepted_raw_id, - accepted_source_revision, accepted_content_hash, accepted_frontier_kind, - accepted_frontier, acquisition_generation, decided_at_ms - FROM raw_revision_heads - """ - ) - source.commit() - index.commit() - - report = repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]) - assert report.ineligible_count == 1 - - -def test_browser_capture_origin_rejects_unresolved_source_membership(tmp_path: Path) -> None: - raw_id = _seed_mismatched_browser_head(tmp_path) - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute( - "UPDATE raw_session_memberships SET decision = 'ambiguous', decided_at_ms = 2 WHERE raw_id = ?", - (raw_id,), - ) - - report = repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]) - - assert report.ineligible_count == 1 - - -def test_browser_capture_origin_rejects_legacy_raw_without_native_id(tmp_path: Path) -> None: - raw_id = _seed_mismatched_browser_head(tmp_path) - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute("UPDATE raw_sessions SET native_id = NULL WHERE raw_id = ?", (raw_id,)) - - report = repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]) - - assert report.ineligible_count == 1 - assert report.items[0].reason == "source envelope does not exactly bind the normalized session" - - -def test_legacy_browser_native_id_copy_forward_preserves_evidence_and_is_idempotent(tmp_path: Path) -> None: - raw_id = _seed_legacy_browser_head_without_native_id(tmp_path) - ordinary = repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]) - assert ordinary.ineligible_count == 1 - with ( - closing(sqlite3.connect(tmp_path / "source.db")) as source, - closing(sqlite3.connect(tmp_path / "index.db")) as index, - ): - old_source = source.execute("SELECT * FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone() - old_refs = source.execute("SELECT * FROM blob_refs WHERE ref_id = ? ORDER BY ref_type", (raw_id,)).fetchall() - old_memberships = source.execute( - "SELECT * FROM raw_session_memberships WHERE raw_id = ? ORDER BY logical_source_key", (raw_id,) - ).fetchall() - old_head = index.execute( - "SELECT * FROM raw_revision_heads WHERE logical_source_key = 'unknown:browser-origin-one'" - ).fetchone() - old_applications = index.execute( - "SELECT * FROM raw_revision_applications WHERE raw_id = ? ORDER BY decision_id", (raw_id,) - ).fetchall() - dry_run = repair_legacy_browser_capture_missing_native_ids(_config(tmp_path), [raw_id]) - item = dry_run.items[0] - assert item.status == "eligible" - assert item.legacy_null_native_id is True - assert item.parser_derived_native_id == "browser-origin-one" - receipt = tmp_path / "legacy-native-id.jsonl" - applied = repair_legacy_browser_capture_missing_native_ids( - _config(tmp_path), [raw_id], apply=True, receipt_path=receipt, proof_digest=dry_run.proof_digest - ) - assert applied.repaired_count == 1 - copy_raw_id = applied.items[0].copy_forward_raw_id - assert copy_raw_id is not None - lines = [json.loads(line) for line in receipt.read_text().splitlines()] - assert lines[0]["schema"] == "polylogue.browser-capture-legacy-native-id-copy-forward.v1" - assert lines[0]["transaction_protocol"] == "rollback-superjournal-v1" - assert lines[0]["targets"][0]["legacy_null_native_id"] is True - assert lines[0]["targets"][0]["parser_derived_native_id"] == "browser-origin-one" - assert lines[-1]["legacy_native_witness_bindings"] == [ - {"raw_id": raw_id, "legacy_null_native_id": True, "parser_derived_native_id": "browser-origin-one"} - ] - assert lines[-1]["transaction_protocol"] == "rollback-superjournal-v1" - assert lines[-1]["legacy_journal_modes"]["after_restore"] == {"source": "wal", "index": "wal"} - assert _journal_modes(tmp_path) == {"source": "wal", "index": "wal"} - with ( - closing(sqlite3.connect(tmp_path / "source.db")) as source, - closing(sqlite3.connect(tmp_path / "index.db")) as index, - ): - assert source.execute("SELECT * FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone() == old_source - assert ( - source.execute("SELECT * FROM blob_refs WHERE ref_id = ? ORDER BY ref_type", (raw_id,)).fetchall() - == old_refs - ) - assert ( - source.execute( - "SELECT * FROM raw_session_memberships WHERE raw_id = ? ORDER BY logical_source_key", (raw_id,) - ).fetchall() - == old_memberships - ) - assert ( - index.execute( - "SELECT * FROM raw_revision_heads WHERE logical_source_key = 'unknown:browser-origin-one'" - ).fetchone() - == old_head - ) - assert ( - index.execute( - "SELECT * FROM raw_revision_applications WHERE raw_id = ? ORDER BY decision_id", (raw_id,) - ).fetchall() - == old_applications - ) - assert source.execute( - "SELECT origin, native_id FROM raw_sessions WHERE raw_id = ?", (copy_raw_id,) - ).fetchone() == ( - "chatgpt-export", - "browser-origin-one", - ) - reapplied = repair_legacy_browser_capture_missing_native_ids( - _config(tmp_path), [raw_id], apply=True, receipt_path=receipt, proof_digest=dry_run.proof_digest - ) - assert reapplied.already_repaired_count == 1 - - -@pytest.mark.parametrize("checkpoint", ["before_commit", "after_commit"]) -def test_legacy_browser_native_id_crash_boundary_recovers_planned_receipt(tmp_path: Path, checkpoint: str) -> None: - raw_id = _seed_legacy_browser_head_without_native_id(tmp_path) - dry_run = repair_legacy_browser_capture_missing_native_ids(_config(tmp_path), [raw_id]) - before = { - (tier, table): _rows(tmp_path, tier, table, "1 = 1", ()) - for tier, tables in { - "source": ("raw_sessions", "blob_refs", "raw_session_memberships", "raw_membership_census"), - "index": ("sessions", "raw_revision_heads", "raw_revision_applications"), - }.items() - for table in tables - } - - receipt = tmp_path / f"legacy-crash-{checkpoint}.jsonl" - program = f""" -import os -from pathlib import Path - -from polylogue.config import Config -import polylogue.storage.repair as repair - -root = Path({str(tmp_path)!r}) - -def crash_at_selected_boundary(stage: str) -> None: - if stage == {checkpoint!r}: - os._exit(87) + preview = inspect_raw_authority_frontier(_config(tmp_path)) + selected = next(item for item in preview.items if item.raw_id == raw_id) -repair._legacy_browser_copy_forward_checkpoint = crash_at_selected_boundary -repair.repair_legacy_browser_capture_missing_native_ids( - Config(archive_root=root, render_root=root / 'render', sources=[], db_path=root / 'index.db'), - [{raw_id!r}], - apply=True, - receipt_path=Path({str(receipt)!r}), - proof_digest={dry_run.proof_digest!r}, -) -""" - child = subprocess.run( - [sys.executable, "-c", program], - check=False, - capture_output=True, - text=True, - timeout=30, - ) - assert child.returncode == 87, child.stderr - assert [json.loads(line)["state"] for line in receipt.read_text().splitlines()] == ["planned"] - assert _journal_modes(tmp_path) == {"source": "delete", "index": "delete"} - if checkpoint == "before_commit": - for (tier, table), rows in before.items(): - assert _rows(tmp_path, tier, table, "1 = 1", ()) == rows - else: - assert _rows(tmp_path, "source", "raw_sessions", "raw_id != ?", (raw_id,)) - assert _rows(tmp_path, "index", "raw_revision_heads", "logical_source_key LIKE ?", ("chatgpt:%",)) - - resumed = repair_legacy_browser_capture_missing_native_ids( - _config(tmp_path), [raw_id], apply=True, receipt_path=receipt, proof_digest=dry_run.proof_digest - ) - - assert resumed.already_repaired_count == 1 - assert _journal_modes(tmp_path) == {"source": "wal", "index": "wal"} - lines = [json.loads(line) for line in receipt.read_text().splitlines()] - assert [line["state"] for line in lines] == ["planned", "applied"] - assert lines[-1]["transaction_protocol"] == "rollback-superjournal-v1" - - -@pytest.mark.parametrize("mutation", ["native", "path", "blob_ref", "census", "head", "application"]) -def test_legacy_browser_native_id_copy_forward_rejects_any_witness_drift(tmp_path: Path, mutation: str) -> None: - raw_id = _seed_legacy_browser_head_without_native_id(tmp_path) - if mutation == "native": - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute("UPDATE raw_sessions SET native_id = 'wrong-native' WHERE raw_id = ?", (raw_id,)) - elif mutation == "path": - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute( - "UPDATE raw_sessions SET source_path = 'browser-capture/chatgpt/wrong.json' WHERE raw_id = ?", (raw_id,) - ) - elif mutation == "blob_ref": - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute("DELETE FROM blob_refs WHERE ref_id = ?", (raw_id,)) - elif mutation == "census": - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute("UPDATE raw_membership_census SET member_count = 2 WHERE raw_id = ?", (raw_id,)) - elif mutation == "head": - with sqlite3.connect(tmp_path / "index.db") as index: - index.execute( - "UPDATE raw_revision_heads SET accepted_frontier = accepted_frontier + 1 WHERE logical_source_key = 'unknown:browser-origin-one'" - ) - else: - with sqlite3.connect(tmp_path / "index.db") as index: - index.execute("UPDATE raw_revision_applications SET decision = 'superseded' WHERE raw_id = ?", (raw_id,)) - - report = repair_legacy_browser_capture_missing_native_ids(_config(tmp_path), [raw_id]) - - assert report.ineligible_count == 1 - - -def test_legacy_browser_native_id_copy_forward_accepts_source_v7(tmp_path: Path) -> None: - raw_id = _seed_legacy_browser_head_without_native_id(tmp_path) - with closing(sqlite3.connect(tmp_path / "source.db")) as source: - source.execute("ALTER TABLE raw_sessions DROP COLUMN capture_mode") - source.execute("PRAGMA user_version = 7") - dry_run = repair_legacy_browser_capture_missing_native_ids(_config(tmp_path), [raw_id]) - applied = repair_legacy_browser_capture_missing_native_ids( + report = apply_raw_authority_frontier( _config(tmp_path), - [raw_id], - apply=True, - receipt_path=tmp_path / "legacy-source-v7.jsonl", - proof_digest=dry_run.proof_digest, + preview_census_id=preview.census_id, + selected_plan_ids=(selected.plan_id,), ) - assert applied.repaired_count == 1 - -def test_legacy_browser_native_id_copy_forward_rejects_stale_proof_before_writing(tmp_path: Path) -> None: - raw_id = _seed_legacy_browser_head_without_native_id(tmp_path) - with sqlite3.connect(tmp_path / "source.db") as source: - before = source.execute("SELECT COUNT(*) FROM raw_sessions").fetchone()[0] - with pytest.raises(RuntimeError, match="proof digest"): - repair_legacy_browser_capture_missing_native_ids( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=tmp_path / "stale-proof.jsonl", - proof_digest="0" * 64, - ) - with sqlite3.connect(tmp_path / "source.db") as source: - assert source.execute("SELECT COUNT(*) FROM raw_sessions").fetchone()[0] == before - - -def test_legacy_browser_native_id_copy_forward_refuses_preexisting_canonical_head(tmp_path: Path) -> None: - raw_id = _seed_legacy_browser_head_without_native_id(tmp_path) - _seed_equivalent_canonical_head(tmp_path, raw_id) - - report = repair_legacy_browser_capture_missing_native_ids(_config(tmp_path), [raw_id]) - - assert report.ineligible_count == 1 - assert report.items[0].reason == "legacy-native-id copy-forward refuses pre-existing canonical head authority" - - -def test_legacy_browser_native_id_copy_forwards_from_semantic_canonical_witness(tmp_path: Path) -> None: - raw_id = _seed_legacy_browser_head_without_native_id(tmp_path) - semantic_raw_id = _seed_semantic_canonical_head(tmp_path, raw_id) - sibling_raw_id = _seed_semantic_superseded_sibling(tmp_path, semantic_raw_id) - active_index = _stage_active_index_generation(tmp_path) - historical_before = _rows( - tmp_path, - "source", - "raw_sessions", - "raw_id IN (?, ?)", - (semantic_raw_id, sibling_raw_id), - ) - with closing(sqlite3.connect(tmp_path / "source.db")) as source: - source_count = source.execute("SELECT COUNT(*) FROM raw_sessions").fetchone()[0] - - dry_run = repair_legacy_browser_capture_missing_native_ids(_config(tmp_path), [raw_id]) - - assert dry_run.eligible_count == 1, dry_run.items[0].reason - assert dry_run.items[0].legacy_null_native_id is True - assert dry_run.items[0].repair_strategy == "copy_forward" - assert dry_run.items[0].semantic_canonical_raw_id == semantic_raw_id - assert dry_run.items[0].semantic_historical_raw_ids == (sibling_raw_id,) - assert dry_run.items[0].semantic_witness_digest is not None - assert (tmp_path / "index.db").resolve() == active_index - receipt = tmp_path / "legacy-semantic-copy-receipt.jsonl" - applied = repair_legacy_browser_capture_missing_native_ids( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - - copy_raw_id = applied.items[0].copy_forward_raw_id - assert copy_raw_id is not None - with sqlite3.connect(tmp_path / "source.db") as source: - assert source.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (source_count + 1,) - assert source.execute("SELECT native_id FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone() == (None,) - assert ( - _rows(tmp_path, "source", "raw_sessions", "raw_id IN (?, ?)", (semantic_raw_id, sibling_raw_id)) - == historical_before - ) - lines = [json.loads(line) for line in receipt.read_text().splitlines()] - assert lines[-1]["transaction_protocol"] == "rollback-superjournal-v1" - assert lines[-1]["semantic_witness_bindings"][0]["semantic_canonical_raw_id"] == semantic_raw_id - assert lines[-1]["semantic_witness_bindings"][0]["semantic_historical_raw_ids"] == [sibling_raw_id] - with closing(sqlite3.connect(tmp_path / "index.db")) as index: - assert index.execute( - "SELECT raw_id FROM sessions WHERE session_id = 'chatgpt-export:browser-origin-one'" - ).fetchone() == (copy_raw_id,) - - -def test_legacy_browser_native_id_rejects_changed_semantic_witness_before_writing(tmp_path: Path) -> None: - raw_id = _seed_legacy_browser_head_without_native_id(tmp_path) - semantic_raw_id = _seed_semantic_canonical_head(tmp_path, raw_id) - dry_run = repair_legacy_browser_capture_missing_native_ids(_config(tmp_path), [raw_id]) - with closing(sqlite3.connect(tmp_path / "index.db")) as index, index: - index.execute( - "UPDATE raw_revision_heads SET accepted_frontier = accepted_frontier + 1 WHERE accepted_raw_id = ?", - (semantic_raw_id,), - ) - - with pytest.raises(RuntimeError, match="proof digest"): - repair_legacy_browser_capture_missing_native_ids( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=tmp_path / "legacy-semantic-stale-proof.jsonl", - proof_digest=dry_run.proof_digest, - ) - assert ( - _rows( - tmp_path, - "source", - "raw_sessions", - "source_path LIKE ?", - ("browser-capture-origin-copy-forward/%",), - ) - == [] - ) - - -def test_legacy_browser_native_id_copy_forward_requires_single_membership_key(tmp_path: Path) -> None: - raw_id = _seed_legacy_browser_head_without_native_id(tmp_path) - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute( - """ - INSERT INTO raw_session_memberships ( - raw_id, logical_source_key, provider_session_id, source_revision, - normalized_content_hash, message_count, acquisition_generation, - revision_authority, decision, decided_at_ms - ) - SELECT raw_id, 'chatgpt:competing-membership', provider_session_id, - source_revision, normalized_content_hash, message_count, - acquisition_generation, revision_authority, decision, decided_at_ms - FROM raw_session_memberships - WHERE raw_id = ? - """, - (raw_id,), - ) - - report = repair_legacy_browser_capture_missing_native_ids(_config(tmp_path), [raw_id]) - - assert report.ineligible_count == 1 - assert report.items[0].reason == "membership census does not exactly reproduce the accepted session" - - -def test_legacy_browser_native_id_copy_forward_requires_single_payload_blob_ref(tmp_path: Path) -> None: - raw_id = _seed_legacy_browser_head_without_native_id(tmp_path) - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute( - """ - INSERT INTO blob_refs (blob_hash, ref_id, ref_type, source_path, size_bytes, acquired_at_ms) - SELECT x'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', - ref_id, ref_type, source_path, size_bytes, acquired_at_ms - FROM blob_refs WHERE ref_id = ? AND ref_type = 'raw_payload' - """, - (raw_id,), - ) - - report = repair_legacy_browser_capture_missing_native_ids(_config(tmp_path), [raw_id]) - - assert report.ineligible_count == 1 - assert report.items[0].reason == "source envelope does not exactly bind the normalized session" - - -def test_legacy_browser_native_id_atomic_apply_rolls_back_after_source_stage( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - import polylogue.storage.repair as repair_module - - raw_id = _seed_legacy_browser_head_without_native_id(tmp_path) - dry_run = repair_legacy_browser_capture_missing_native_ids(_config(tmp_path), [raw_id]) - tables = { - "source": ("raw_sessions", "blob_refs", "raw_session_memberships", "raw_membership_census"), - "index": ("sessions", "raw_revision_heads", "raw_revision_applications"), - } - before = { - (tier, table): _rows(tmp_path, tier, table, "1 = 1", ()) - for tier, tier_tables in tables.items() - for table in tier_tables + assert report.executed_plan_count == 1 + assert report.retryable_plan_count == 0 + postflight = inspect_raw_authority_frontier(_config(tmp_path)) + assert all(item.raw_id != raw_id or item.state is RawAuthorityFrontierState.SUPERSEDED for item in postflight.items) + assert set(postflight.state_counts) <= { + RawAuthorityFrontierState.PROVEN_CURRENT.value, + RawAuthorityFrontierState.SUPERSEDED.value, } - - def fail_after_source_stage(conn: sqlite3.Connection, item: object) -> None: - raise RuntimeError("injected index transition failure") - - monkeypatch.setattr(repair_module, "_apply_browser_origin_repair_item", fail_after_source_stage) - receipt = tmp_path / "legacy-atomic-rollback.jsonl" - with pytest.raises(RuntimeError, match="injected index transition failure"): - repair_legacy_browser_capture_missing_native_ids( - _config(tmp_path), [raw_id], apply=True, receipt_path=receipt, proof_digest=dry_run.proof_digest - ) - - assert [json.loads(line)["state"] for line in receipt.read_text().splitlines()] == ["planned"] - for (tier, table), rows in before.items(): - assert _rows(tmp_path, tier, table, "1 = 1", ()) == rows - - -def test_browser_capture_origin_copy_forward_accepts_source_v7_without_capture_mode(tmp_path: Path) -> None: - raw_id = _seed_mismatched_browser_head(tmp_path) - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute("ALTER TABLE raw_sessions DROP COLUMN capture_mode") - source.execute("PRAGMA user_version = 7") - dry_run = repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]) - receipt = tmp_path / "source-v7-receipt.jsonl" - - applied = repair_browser_capture_origin_mismatches( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - - assert applied.repaired_count == 1 - assert applied.items[0].status == "already_repaired" + assert not (tmp_path / "recovery").exists() -def test_browser_capture_origin_repair_restores_equivalent_canonical_head(tmp_path: Path) -> None: +def test_unified_frontier_restores_equivalent_canonical_browser_head(tmp_path: Path) -> None: mismatched_raw_id = _seed_mismatched_browser_head(tmp_path) canonical_raw_id = _seed_equivalent_canonical_head(tmp_path, mismatched_raw_id) - with sqlite3.connect(tmp_path / "source.db") as source: - source_count = source.execute("SELECT COUNT(*) FROM raw_sessions").fetchone()[0] - dry_run = repair_browser_capture_origin_mismatches(_config(tmp_path), [mismatched_raw_id]) + preview = inspect_raw_authority_frontier(_config(tmp_path)) + selected = next(item for item in preview.items if item.raw_id == mismatched_raw_id) + strategy_item = cast(dict[str, Any], selected.strategy_witness["item"]) + assert strategy_item["repair_strategy"] == "restore_canonical_head" - assert dry_run.items[0].repair_strategy == "restore_canonical_head" - assert dry_run.items[0].replacement_raw_id == canonical_raw_id - applied = repair_browser_capture_origin_mismatches( + report = apply_raw_authority_frontier( _config(tmp_path), - [mismatched_raw_id], - apply=True, - receipt_path=tmp_path / "restore-receipt.jsonl", - proof_digest=dry_run.proof_digest, + preview_census_id=preview.census_id, + selected_plan_ids=(selected.plan_id,), ) - assert applied.repaired_count == 1 - with sqlite3.connect(tmp_path / "source.db") as source: - assert source.execute("SELECT COUNT(*) FROM raw_sessions").fetchone()[0] == source_count + assert report.executed_plan_count == 1 + assert report.retryable_plan_count == 0 with sqlite3.connect(tmp_path / "index.db") as index: assert index.execute( "SELECT raw_id FROM sessions WHERE session_id = 'chatgpt-export:browser-origin-one'" ).fetchone() == (canonical_raw_id,) - assert index.execute( - """ - SELECT accepted_raw_id, detail FROM raw_revision_applications - WHERE raw_id = ? AND logical_source_key = 'chatgpt:browser-origin-one' - AND decision = 'superseded' - """, - (mismatched_raw_id,), - ).fetchone() == ( - canonical_raw_id, - f"browser_capture_origin_supersession:{mismatched_raw_id}", - ) - - -def test_browser_capture_origin_repair_copy_forwards_from_semantic_canonical_witness(tmp_path: Path) -> None: - mismatched_raw_id = _seed_mismatched_browser_head(tmp_path) - semantic_raw_id = _seed_semantic_canonical_head(tmp_path, mismatched_raw_id) - with sqlite3.connect(tmp_path / "source.db") as source: - source_count = source.execute("SELECT COUNT(*) FROM raw_sessions").fetchone()[0] - - dry_run = repair_browser_capture_origin_mismatches(_config(tmp_path), [mismatched_raw_id]) - - assert dry_run.eligible_count == 1, dry_run.items[0].reason - assert dry_run.items[0].repair_strategy == "copy_forward" - assert dry_run.items[0].replacement_raw_id != semantic_raw_id - applied = repair_browser_capture_origin_mismatches( - _config(tmp_path), - [mismatched_raw_id], - apply=True, - receipt_path=tmp_path / "semantic-copy-receipt.jsonl", - proof_digest=dry_run.proof_digest, - ) - - copy_raw_id = applied.items[0].copy_forward_raw_id - assert copy_raw_id is not None - with sqlite3.connect(tmp_path / "source.db") as source: - assert source.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (source_count + 1,) - assert source.execute( - "SELECT revision_authority FROM raw_sessions WHERE raw_id = ?", (semantic_raw_id,) - ).fetchone() == ("quarantined",) - with sqlite3.connect(tmp_path / "index.db") as index: - assert index.execute( - "SELECT raw_id FROM sessions WHERE session_id = 'chatgpt-export:browser-origin-one'" - ).fetchone() == (copy_raw_id,) - - -def test_browser_origin_repair_accepts_exact_semantic_superseded_sibling(tmp_path: Path) -> None: - mismatched_raw_id = _seed_mismatched_browser_head(tmp_path) - semantic_raw_id = _seed_semantic_canonical_head(tmp_path, mismatched_raw_id) - sibling_raw_id = _seed_semantic_superseded_sibling(tmp_path, semantic_raw_id) - with sqlite3.connect(tmp_path / "source.db") as source: - assert ( - source.execute( - "SELECT blob_hash FROM raw_sessions WHERE raw_id = ?", - (sibling_raw_id,), - ).fetchone() - != source.execute( - "SELECT blob_hash FROM raw_sessions WHERE raw_id = ?", - (semantic_raw_id,), - ).fetchone() - ) - historical_before = _rows( - tmp_path, - "index", - "raw_revision_applications", - "raw_id = ?", - (sibling_raw_id,), - ) - - dry_run = repair_browser_capture_origin_mismatches(_config(tmp_path), [mismatched_raw_id]) - - assert dry_run.eligible_count == 1, dry_run.items[0].reason - item = dry_run.items[0] - assert item.semantic_canonical_raw_id == semantic_raw_id - assert item.semantic_historical_raw_ids == (sibling_raw_id,) - assert item.semantic_witness_digest is not None - assert ( - _rows( - tmp_path, - "index", - "raw_revision_applications", - "raw_id = ?", - (sibling_raw_id,), - ) - == historical_before - ) - receipt = tmp_path / "semantic-sibling-receipt.jsonl" - applied = repair_browser_capture_origin_mismatches( - _config(tmp_path), - [mismatched_raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - assert applied.repaired_count == 1 - assert applied.items[0].status == "already_repaired" - assert applied.items[0].semantic_canonical_raw_id == semantic_raw_id - assert applied.items[0].semantic_historical_raw_ids == (sibling_raw_id,) - assert applied.items[0].semantic_witness_digest == item.semantic_witness_digest - assert applied.items[0].terminal_byte_witness_digest is not None - - reapplied = repair_browser_capture_origin_mismatches( - _config(tmp_path), - [mismatched_raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - - assert reapplied.repaired_count == 0 - assert reapplied.items[0].semantic_witness_digest == item.semantic_witness_digest - assert [json.loads(line)["state"] for line in receipt.read_text().splitlines()] == ["planned", "applied"] - - -@pytest.mark.parametrize( - "mutation", - [ - "head_frontier", - "head_revision", - "receipt_id", - "receipt_time", - "copy_predecessor_revision", - "copy_predecessor_raw", - "copy_append_start", - "copy_append_end", - "copy_capture_mode", - ], -) -def test_browser_origin_repair_refuses_mutated_terminal_byte_authority(tmp_path: Path, mutation: str) -> None: - mismatched_raw_id = _seed_mismatched_browser_head(tmp_path) - semantic_raw_id = _seed_semantic_canonical_head(tmp_path, mismatched_raw_id) - _seed_semantic_superseded_sibling(tmp_path, semantic_raw_id) - dry_run = repair_browser_capture_origin_mismatches(_config(tmp_path), [mismatched_raw_id]) - receipt = tmp_path / "terminal-byte-authority.jsonl" - applied = repair_browser_capture_origin_mismatches( - _config(tmp_path), - [mismatched_raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - copy_raw_id = applied.items[0].copy_forward_raw_id - assert copy_raw_id is not None - with sqlite3.connect(tmp_path / "source.db") as source: - source_count = source.execute("SELECT COUNT(*) FROM raw_sessions").fetchone()[0] - with sqlite3.connect(tmp_path / "index.db") as index: - if mutation == "head_frontier": - index.execute( - "UPDATE raw_revision_heads SET accepted_frontier = accepted_frontier + 1 WHERE accepted_raw_id = ?", - (copy_raw_id,), - ) - elif mutation == "head_revision": - index.execute( - "UPDATE raw_revision_heads SET accepted_source_revision = ? WHERE accepted_raw_id = ?", - ("0" * 64, copy_raw_id), - ) - elif mutation == "receipt_id": - index.execute( - "UPDATE raw_revision_applications SET decision_id = ? WHERE raw_id = ?", - ("e" * 64, copy_raw_id), - ) - elif mutation == "receipt_time": - index.execute( - "UPDATE raw_revision_applications SET decided_at_ms = decided_at_ms + 1 WHERE raw_id = ?", - (copy_raw_id,), - ) - else: - with sqlite3.connect(tmp_path / "source.db") as source: - if mutation == "copy_predecessor_revision": - source.execute( - "UPDATE raw_sessions SET predecessor_source_revision = ? WHERE raw_id = ?", - ("0" * 64, copy_raw_id), - ) - elif mutation == "copy_predecessor_raw": - source.execute( - "UPDATE raw_sessions SET predecessor_raw_id = ? WHERE raw_id = ?", - ("f" * 64, copy_raw_id), - ) - elif mutation == "copy_append_start": - source.execute("UPDATE raw_sessions SET append_start_offset = 0 WHERE raw_id = ?", (copy_raw_id,)) - elif mutation == "copy_append_end": - source.execute("UPDATE raw_sessions SET append_end_offset = 1 WHERE raw_id = ?", (copy_raw_id,)) - else: - source.execute("UPDATE raw_sessions SET capture_mode = 'unknown' WHERE raw_id = ?", (copy_raw_id,)) - - with pytest.raises(RuntimeError, match="ineligible"): - repair_browser_capture_origin_mismatches( - _config(tmp_path), - [mismatched_raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - with sqlite3.connect(tmp_path / "source.db") as source: - assert source.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (source_count,) - assert [json.loads(line)["state"] for line in receipt.read_text().splitlines()] == ["planned", "applied"] - - -def test_browser_origin_repair_refuses_competing_terminal_copy_application(tmp_path: Path) -> None: - raw_id = _seed_mismatched_browser_head(tmp_path) - dry_run = repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]) - receipt = tmp_path / "competing-copy-application.jsonl" - applied = repair_browser_capture_origin_mismatches( - _config(tmp_path), [raw_id], apply=True, receipt_path=receipt, proof_digest=dry_run.proof_digest - ) - item = applied.items[0] - assert item.copy_forward_raw_id is not None - assert item.session_id is not None - assert item.canonical_logical_source_key is not None - assert item.blob_hash is not None - with sqlite3.connect(tmp_path / "source.db") as source: - source_count = source.execute("SELECT COUNT(*) FROM raw_sessions").fetchone()[0] - with sqlite3.connect(tmp_path / "index.db") as index: - record_revision_application_sync( - index, - RevisionApplicationReceipt( - raw_id=item.copy_forward_raw_id, - session_id=item.session_id, - logical_source_key=item.canonical_logical_source_key, - source_revision=item.blob_hash, - acquisition_generation=0, - decision=ApplicationDecision.DEFERRED, - accepted_raw_id=None, - accepted_source_revision=None, - accepted_content_hash=None, - detail="competing copy receipt", - ), - decided_at_ms=99, - ) - - with pytest.raises(RuntimeError, match="ineligible"): - repair_browser_capture_origin_mismatches( - _config(tmp_path), [raw_id], apply=True, receipt_path=receipt, proof_digest=dry_run.proof_digest - ) - with sqlite3.connect(tmp_path / "source.db") as source: - assert source.execute("SELECT COUNT(*) FROM raw_sessions").fetchone() == (source_count,) - assert [json.loads(line)["state"] for line in receipt.read_text().splitlines()] == ["planned", "applied"] - - -@pytest.mark.parametrize( - "decision", [ApplicationDecision.SUPERSEDED, ApplicationDecision.DEFERRED, ApplicationDecision.AMBIGUOUS] -) -def test_browser_origin_repair_refuses_extra_old_key_receipt(tmp_path: Path, decision: ApplicationDecision) -> None: - raw_id = _seed_mismatched_browser_head(tmp_path) - with sqlite3.connect(tmp_path / "source.db") as source: - blob_hash = source.execute( - "SELECT lower(hex(blob_hash)) FROM raw_sessions WHERE raw_id = ?", (raw_id,) - ).fetchone()[0] - with sqlite3.connect(tmp_path / "index.db") as index: - accepted_hash = bytes( - index.execute( - "SELECT content_hash FROM sessions WHERE session_id = 'chatgpt-export:browser-origin-one'" - ).fetchone()[0] - ) - record_revision_application_sync( - index, - RevisionApplicationReceipt( - raw_id=raw_id, - session_id="chatgpt-export:browser-origin-one", - logical_source_key="unknown:browser-origin-one", - source_revision=blob_hash, - acquisition_generation=0, - decision=decision, - accepted_raw_id=raw_id if decision is ApplicationDecision.SUPERSEDED else None, - accepted_source_revision=blob_hash if decision is ApplicationDecision.SUPERSEDED else None, - accepted_content_hash=accepted_hash if decision is ApplicationDecision.SUPERSEDED else None, - detail=f"unexpected {decision.value}", - ), - decided_at_ms=98, - ) - - assert repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]).ineligible_count == 1 - - -@pytest.mark.parametrize("head_shape", ["old", "semantic"]) -def test_browser_origin_repair_refuses_head_receipt_timestamp_drift(tmp_path: Path, head_shape: str) -> None: - raw_id = _seed_mismatched_browser_head(tmp_path) - accepted_raw_id = raw_id if head_shape == "old" else _seed_semantic_canonical_head(tmp_path, raw_id) - with sqlite3.connect(tmp_path / "index.db") as index: - index.execute( - "UPDATE raw_revision_heads SET decided_at_ms = decided_at_ms + 1 WHERE accepted_raw_id = ?", - (accepted_raw_id,), - ) - - assert repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]).ineligible_count == 1 - - -@pytest.mark.parametrize( - "mutation", - [ - "receipt", - "revision", - "membership", - "selected", - "blob", - "native_id", - "source_index", - "blob_ref_path", - "predecessor_source", - "predecessor_raw", - "append_start", - "append_end", - "capture_mode", - "negative_decided_at", - ], -) -def test_browser_origin_repair_rejects_underproven_semantic_supersession(tmp_path: Path, mutation: str) -> None: - mismatched_raw_id = _seed_mismatched_browser_head(tmp_path) - semantic_raw_id = _seed_semantic_canonical_head(tmp_path, mismatched_raw_id) - sibling_raw_id = _seed_semantic_superseded_sibling(tmp_path, semantic_raw_id) - if mutation == "receipt": - with sqlite3.connect(tmp_path / "index.db") as index: - index.execute( - "UPDATE raw_revision_applications SET accepted_raw_id = ? WHERE raw_id = ?", - (sibling_raw_id, sibling_raw_id), - ) - elif mutation == "revision": - with sqlite3.connect(tmp_path / "index.db") as index: - index.execute( - "UPDATE raw_revision_applications SET accepted_source_revision = ? WHERE raw_id = ?", - ("0" * 64, sibling_raw_id), - ) - elif mutation == "membership": - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute( - "UPDATE raw_session_memberships SET decision = 'ambiguous', decided_at_ms = 4 WHERE raw_id = ?", - (sibling_raw_id,), - ) - elif mutation == "selected": - with sqlite3.connect(tmp_path / "index.db") as index: - index.execute( - "UPDATE raw_revision_applications SET decision = 'selected_baseline' WHERE raw_id = ?", - (sibling_raw_id,), - ) - elif mutation == "blob": - different_payload = _browser_payload("browser-origin-two") - blob_hash, blob_size = BlobStore(tmp_path / "blob").write_from_bytes(different_payload) - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute( - "UPDATE raw_sessions SET native_id = 'browser-origin-two', blob_hash = ?, blob_size = ? WHERE raw_id = ?", - (bytes.fromhex(blob_hash), blob_size, sibling_raw_id), - ) - source.execute( - "UPDATE blob_refs SET blob_hash = ?, size_bytes = ? WHERE ref_id = ?", - (bytes.fromhex(blob_hash), blob_size, sibling_raw_id), - ) - elif mutation == "negative_decided_at": - with sqlite3.connect(tmp_path / "index.db") as index: - index.execute("PRAGMA ignore_check_constraints = ON") - index.execute( - "UPDATE raw_revision_applications SET decided_at_ms = -1 WHERE raw_id = ?", - (sibling_raw_id,), - ) - else: - updates = { - "native_id": ("native_id = 'wrong-native-id'", ()), - "source_index": ("source_index = 1", ()), - "blob_ref_path": (None, ()), - "predecessor_source": ("predecessor_source_revision = ?", ("0" * 64,)), - "predecessor_raw": ("predecessor_raw_id = ?", ("f" * 64,)), - "append_start": ("append_start_offset = 0", ()), - "append_end": ("append_end_offset = 1", ()), - "capture_mode": ("capture_mode = 'unknown'", ()), - } - assignment, params = updates[mutation] - with sqlite3.connect(tmp_path / "source.db") as source: - if assignment is None: - source.execute( - "UPDATE blob_refs SET source_path = 'browser-capture/wrong.json' WHERE ref_id = ?", - (sibling_raw_id,), - ) - else: - source.execute(f"UPDATE raw_sessions SET {assignment} WHERE raw_id = ?", (*params, sibling_raw_id)) - - report = repair_browser_capture_origin_mismatches(_config(tmp_path), [mismatched_raw_id]) - - assert report.ineligible_count == 1 - - -@pytest.mark.parametrize("field", ["decision_id", "detail", "decided_at_ms", "frontier"]) -def test_browser_origin_repair_refuses_changed_semantic_receipt_before_apply(tmp_path: Path, field: str) -> None: - mismatched_raw_id = _seed_mismatched_browser_head(tmp_path) - semantic_raw_id = _seed_semantic_canonical_head(tmp_path, mismatched_raw_id) - sibling_raw_id = _seed_semantic_superseded_sibling(tmp_path, semantic_raw_id) - dry_run = repair_browser_capture_origin_mismatches(_config(tmp_path), [mismatched_raw_id]) - with sqlite3.connect(tmp_path / "index.db") as index: - if field == "decision_id": - index.execute( - "UPDATE raw_revision_applications SET decision_id = ? WHERE raw_id = ?", - ("f" * 64, sibling_raw_id), - ) - elif field == "detail": - index.execute( - "UPDATE raw_revision_applications SET detail = 'tampered' WHERE raw_id = ?", - (sibling_raw_id,), - ) - elif field == "decided_at_ms": - index.execute( - "UPDATE raw_revision_applications SET decided_at_ms = decided_at_ms + 1 WHERE raw_id = ?", - (sibling_raw_id,), - ) - else: - index.execute( - "UPDATE raw_revision_heads SET accepted_frontier = accepted_frontier + 1 WHERE accepted_raw_id = ?", - (semantic_raw_id,), - ) - - with pytest.raises(RuntimeError, match="(ineligible|proof digest)"): - repair_browser_capture_origin_mismatches( - _config(tmp_path), - [mismatched_raw_id], - apply=True, - receipt_path=tmp_path / f"semantic-{field}.jsonl", - proof_digest=dry_run.proof_digest, - ) - assert ( - _rows(tmp_path, "source", "raw_sessions", "source_path LIKE ?", ("browser-capture-origin-copy-forward/%",)) - == [] - ) - - -@pytest.mark.parametrize( - "mutation", - [ - "frontier", - "pointer", - "membership", - "native_id", - "source_index", - "blob_ref_path", - "predecessor_source", - "predecessor_raw", - "append_start", - "append_end", - "capture_mode", - ], -) -def test_browser_capture_origin_repair_rejects_underproven_semantic_canonical_head( - tmp_path: Path, mutation: str -) -> None: - mismatched_raw_id = _seed_mismatched_browser_head(tmp_path) - semantic_raw_id = _seed_semantic_canonical_head(tmp_path, mismatched_raw_id) - if mutation == "frontier": - with sqlite3.connect(tmp_path / "index.db") as index: - index.execute( - "UPDATE raw_revision_heads SET accepted_frontier_kind = 'byte' WHERE accepted_raw_id = ?", - (semantic_raw_id,), - ) - elif mutation == "pointer": - with sqlite3.connect(tmp_path / "index.db") as index: - index.execute( - "UPDATE sessions SET raw_id = ? WHERE session_id = 'chatgpt-export:browser-origin-one'", - (semantic_raw_id,), - ) - elif mutation == "membership": - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute( - "UPDATE raw_session_memberships SET decision = 'ambiguous', decided_at_ms = 5 WHERE raw_id = ?", - (semantic_raw_id,), - ) - else: - updates = { - "native_id": ("native_id = 'wrong-native-id'", ()), - "source_index": ("source_index = 1", ()), - "blob_ref_path": (None, ()), - "predecessor_source": ("predecessor_source_revision = ?", ("0" * 64,)), - "predecessor_raw": ("predecessor_raw_id = ?", ("f" * 64,)), - "append_start": ("append_start_offset = 0", ()), - "append_end": ("append_end_offset = 1", ()), - "capture_mode": ("capture_mode = 'unknown'", ()), - } - assignment, params = updates[mutation] - with sqlite3.connect(tmp_path / "source.db") as source: - if assignment is None: - source.execute( - "UPDATE blob_refs SET source_path = 'browser-capture/wrong.json' WHERE ref_id = ?", - (semantic_raw_id,), - ) - else: - source.execute(f"UPDATE raw_sessions SET {assignment} WHERE raw_id = ?", (*params, semantic_raw_id)) - - report = repair_browser_capture_origin_mismatches(_config(tmp_path), [mismatched_raw_id]) - - assert report.ineligible_count == 1 - - -@pytest.mark.parametrize( - "mutation", - [ - "envelope", - "membership", - "application", - "native_id", - "source_index", - "blob_ref_path", - "predecessor_source", - "predecessor_raw", - "append_start", - "append_end", - "capture_mode", - ], -) -def test_browser_capture_origin_repair_rejects_underproven_canonical_head(tmp_path: Path, mutation: str) -> None: - mismatched_raw_id = _seed_mismatched_browser_head(tmp_path) - canonical_raw_id = _seed_equivalent_canonical_head(tmp_path, mismatched_raw_id) - if mutation == "envelope": - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute( - "UPDATE raw_sessions SET revision_authority = 'quarantined' WHERE raw_id = ?", (canonical_raw_id,) - ) - elif mutation == "membership": - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute( - "UPDATE raw_session_memberships SET decision = 'ambiguous' WHERE raw_id = ?", (canonical_raw_id,) - ) - elif mutation == "application": - with sqlite3.connect(tmp_path / "index.db") as index: - index.execute( - "DELETE FROM raw_revision_applications WHERE raw_id = ? AND decision = 'selected_baseline'", - (canonical_raw_id,), - ) - else: - updates = { - "native_id": ("native_id = 'wrong-native-id'", ()), - "source_index": ("source_index = 1", ()), - "blob_ref_path": (None, ()), - "predecessor_source": ("predecessor_source_revision = ?", ("0" * 64,)), - "predecessor_raw": ("predecessor_raw_id = ?", ("f" * 64,)), - "append_start": ("append_start_offset = 0", ()), - "append_end": ("append_end_offset = 1", ()), - "capture_mode": ("capture_mode = 'unknown'", ()), - } - assignment, params = updates[mutation] - with sqlite3.connect(tmp_path / "source.db") as source: - if assignment is None: - source.execute( - "UPDATE blob_refs SET source_path = 'browser-capture/wrong.json' WHERE ref_id = ?", - (canonical_raw_id,), - ) - else: - source.execute(f"UPDATE raw_sessions SET {assignment} WHERE raw_id = ?", (*params, canonical_raw_id)) - - report = repair_browser_capture_origin_mismatches(_config(tmp_path), [mismatched_raw_id]) - - assert report.ineligible_count == 1 - - -def test_browser_capture_origin_repair_rejects_missing_canonical_blob(tmp_path: Path) -> None: - mismatched_raw_id = _seed_mismatched_browser_head(tmp_path) - canonical_raw_id = _seed_equivalent_canonical_head(tmp_path, mismatched_raw_id) - with sqlite3.connect(tmp_path / "source.db") as source: - blob_hash = source.execute( - "SELECT hex(blob_hash) FROM raw_sessions WHERE raw_id = ?", (canonical_raw_id,) - ).fetchone()[0] - BlobStore(tmp_path / "blob").blob_path(blob_hash.lower()).unlink() - - report = repair_browser_capture_origin_mismatches(_config(tmp_path), [mismatched_raw_id]) - - assert report.ineligible_count == 1 - - -@pytest.mark.parametrize( - "mutation", - [ - "blob_size", - "native_id", - "source_index", - "blob_ref_path", - "predecessor_source", - "predecessor_raw", - "append_start", - "append_end", - "capture_mode", - ], -) -def test_browser_capture_origin_copy_forward_reproves_before_source_stage( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mutation: str -) -> None: - import polylogue.storage.repair as repair_module - - raw_id = _seed_mismatched_browser_head(tmp_path) - dry_run = repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]) - original_verify = repair_module._verify_browser_origin_copy_forward_source_stage - - def mutate_at_locked_source_stage( - archive_root: Path, - source: sqlite3.Connection, - item: repair_module.BrowserCaptureOriginRepairItem, - ) -> None: - if mutation == "blob_size": - source.execute("UPDATE raw_sessions SET blob_size = blob_size + 1 WHERE raw_id = ?", (raw_id,)) - elif mutation == "native_id": - source.execute("UPDATE raw_sessions SET native_id = 'wrong-native-id' WHERE raw_id = ?", (raw_id,)) - elif mutation == "source_index": - source.execute("UPDATE raw_sessions SET source_index = 1 WHERE raw_id = ?", (raw_id,)) - elif mutation == "blob_ref_path": - source.execute( - "UPDATE blob_refs SET source_path = 'browser-capture/wrong.json' WHERE ref_id = ?", (raw_id,) - ) - elif mutation == "predecessor_source": - source.execute( - "UPDATE raw_sessions SET predecessor_source_revision = ? WHERE raw_id = ?", ("0" * 64, raw_id) - ) - elif mutation == "predecessor_raw": - source.execute("UPDATE raw_sessions SET predecessor_raw_id = ? WHERE raw_id = ?", ("f" * 64, raw_id)) - elif mutation == "append_start": - source.execute("UPDATE raw_sessions SET append_start_offset = 0 WHERE raw_id = ?", (raw_id,)) - elif mutation == "append_end": - source.execute("UPDATE raw_sessions SET append_end_offset = 1 WHERE raw_id = ?", (raw_id,)) - else: - source.execute("UPDATE raw_sessions SET capture_mode = 'chatgpt' WHERE raw_id = ?", (raw_id,)) - original_verify(archive_root, source, item) - - monkeypatch.setattr( - repair_module, "_verify_browser_origin_copy_forward_source_stage", mutate_at_locked_source_stage - ) - with pytest.raises(RuntimeError, match="source evidence changed"): - repair_browser_capture_origin_mismatches( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=tmp_path / "source-race.jsonl", - proof_digest=dry_run.proof_digest, - ) - assert ( - _rows(tmp_path, "source", "raw_sessions", "source_path LIKE ?", ("browser-capture-origin-copy-forward/%",)) - == [] - ) - - -def test_browser_capture_origin_copy_forward_resumes_after_source_stage_interrupt(tmp_path: Path) -> None: - import polylogue.storage.repair as repair_module - - raw_id = _seed_mismatched_browser_head(tmp_path) - dry_run = repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]) - item = dry_run.items[0] - assert item.repair_strategy == "copy_forward" - with sqlite3.connect(tmp_path / "source.db") as source: - repair_module._stage_browser_origin_copy_forward_source(source, item) - - resumed = repair_browser_capture_origin_mismatches( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=tmp_path / "resume-receipt.jsonl", - proof_digest=dry_run.proof_digest, - ) - - assert resumed.repaired_count == 1 - assert resumed.items[0].status == "already_repaired" - assert resumed.items[0].copy_forward_source_complete is True - - -def test_browser_capture_origin_copy_forward_refuses_incomplete_blob_reference(tmp_path: Path) -> None: - import polylogue.storage.repair as repair_module - - raw_id = _seed_mismatched_browser_head(tmp_path) - dry_run = repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]) - item = dry_run.items[0] - assert item.copy_forward_raw_id is not None - with sqlite3.connect(tmp_path / "source.db") as source: - repair_module._stage_browser_origin_copy_forward_source(source, item) - source.execute("DELETE FROM blob_refs WHERE ref_id = ?", (item.copy_forward_raw_id,)) - - with pytest.raises(RuntimeError, match="ineligible"): - repair_browser_capture_origin_mismatches( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=tmp_path / "incomplete-copy-receipt.jsonl", - proof_digest=dry_run.proof_digest, - ) - with sqlite3.connect(tmp_path / "index.db") as index: - assert index.execute( - "SELECT raw_id FROM sessions WHERE session_id = 'chatgpt-export:browser-origin-one'" - ).fetchone() == (raw_id,) - - -def test_browser_capture_origin_live_membership_ignores_quarantined_conflict(tmp_path: Path) -> None: - raw_id = _seed_mismatched_browser_head(tmp_path) - dry_run = repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]) - applied = repair_browser_capture_origin_mismatches( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=tmp_path / "live-membership-repair.jsonl", - proof_digest=dry_run.proof_digest, - ) - copy_raw_id = applied.items[0].copy_forward_raw_id - assert copy_raw_id is not None - session = _parse_one(Provider.CHATGPT, _browser_payload(), "browser-capture/chatgpt/one.json")[0] - cursor = CursorStore(tmp_path / "cursor.sqlite") - processor = LiveBatchProcessor( - cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), - (), - cursor=cursor, - parser_fingerprint="browser-origin-test", - ) - with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: - parsed_by_raw_id = { - raw_id: session, - copy_raw_id: session, - } - projections = {raw_id: session_revision_projection(session), copy_raw_id: session_revision_projection(session)} - # Production guard reproduction: an unresolved quarantined member - # chosen over the existing byte head raises at the real archive writer. - with pytest.raises(RuntimeError, match="membership replay cannot retire"): - archive.apply_raw_membership_classification( - "chatgpt:browser-origin-one", - MembershipClassification((raw_id,), (), ()), - parsed_by_raw_id, - projections, - acquired_at_ms=5, - ) - with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: - processor._apply_membership_sessions(archive, copy_raw_id, [session], acquired_at_ms=6) - assert archive.raw_revision_head_raw_id("chatgpt:browser-origin-one") == copy_raw_id - - -def test_browser_capture_origin_live_membership_defers_all_quarantined_rows(tmp_path: Path) -> None: - raw_id = _seed_mismatched_browser_head(tmp_path) - dry_run = repair_browser_capture_origin_mismatches(_config(tmp_path), [raw_id]) - applied = repair_browser_capture_origin_mismatches( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=tmp_path / "all-quarantined-repair.jsonl", - proof_digest=dry_run.proof_digest, - ) - copy_raw_id = applied.items[0].copy_forward_raw_id - assert copy_raw_id is not None - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute( - "UPDATE raw_session_memberships SET revision_authority = 'quarantined' " - "WHERE logical_source_key = 'chatgpt:browser-origin-one'" - ) - session = _parse_one(Provider.CHATGPT, _browser_payload(), "browser-capture/chatgpt/one.json")[0] - cursor = CursorStore(tmp_path / "all-quarantined-cursor.sqlite") - processor = LiveBatchProcessor( - cast(Any, SimpleNamespace(archive_root=tmp_path, backend=SimpleNamespace(db_path=tmp_path / "index.db"))), - (), - cursor=cursor, - parser_fingerprint="browser-origin-test", - ) - with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: - processor._apply_membership_sessions(archive, copy_raw_id, [session], acquired_at_ms=7) - assert archive.raw_revision_head_raw_id("chatgpt:browser-origin-one") == copy_raw_id - - -@pytest.mark.parametrize( - "native_id", - ( - "no-canonical-773bbbf1", - "no-canonical-bd47782e", - "no-canonical-f43a203a", - ), -) -def test_byte_proven_browser_rekey_copy_forwards_each_no_head_shape(tmp_path: Path, native_id: str) -> None: - raw_id = _seed_byte_proven_browser_head_without_native_id(tmp_path, native_id) - if native_id == "no-canonical-773bbbf1": - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute("ALTER TABLE raw_sessions DROP COLUMN capture_mode") - source.execute("PRAGMA user_version = 7") - active_index = _stage_active_index_generation(tmp_path) - old_evidence = { - ("source", table): _rows( - tmp_path, "source", table, "raw_id = ?" if table != "blob_refs" else "ref_id = ?", (raw_id,) - ) - for table in ("raw_sessions", "blob_refs", "raw_session_memberships", "raw_membership_census") - } | { - ("index", "raw_revision_heads"): _rows( - tmp_path, "index", "raw_revision_heads", "accepted_raw_id = ?", (raw_id,) - ), - ("index", "raw_revision_applications"): _rows( - tmp_path, "index", "raw_revision_applications", "raw_id = ?", (raw_id,) - ), + postflight = inspect_raw_authority_frontier(_config(tmp_path)) + assert set(postflight.state_counts) <= { + RawAuthorityFrontierState.PROVEN_CURRENT.value, + RawAuthorityFrontierState.SUPERSEDED.value, } - dry_run = repair_byte_proven_browser_capture_null_native_ids(_config(tmp_path), [raw_id]) - assert dry_run.eligible_count == 1, dry_run.items[0].reason - assert dry_run.items[0].byte_proven_null_native_id_rekey is True - assert dry_run.items[0].parsed_message_count == 1 - assert (tmp_path / "index.db").resolve() == active_index - receipt = tmp_path / f"byte-rekey-{native_id}.jsonl" - applied = repair_byte_proven_browser_capture_null_native_ids( - _config(tmp_path), [raw_id], apply=True, receipt_path=receipt, proof_digest=dry_run.proof_digest +@pytest.mark.parametrize("authority", ["quarantined", "byte_proven"]) +def test_unified_frontier_admits_historical_null_native_id_strategies(tmp_path: Path, authority: str) -> None: + raw_id = ( + _seed_legacy_browser_head_without_native_id(tmp_path) + if authority == "quarantined" + else _seed_byte_proven_browser_head_without_native_id(tmp_path) ) - copy_raw_id = applied.items[0].copy_forward_raw_id - assert applied.repaired_count == 1 - assert copy_raw_id is not None - for (tier, table), before in old_evidence.items(): - key = "ref_id" if table == "blob_refs" else "accepted_raw_id" if table == "raw_revision_heads" else "raw_id" - assert _rows(tmp_path, tier, table, f"{key} = ?", (raw_id,)) == before - with closing(sqlite3.connect(tmp_path / "source.db")) as source: - assert source.execute( - "SELECT origin, native_id, logical_source_key, revision_authority FROM raw_sessions WHERE raw_id = ?", - (copy_raw_id,), - ).fetchone() == ("chatgpt-export", native_id, f"chatgpt:{native_id}", "byte_proven") - assert source.execute( - "SELECT provider_session_id, message_count, revision_authority, decision FROM raw_session_memberships " - "WHERE raw_id = ?", - (copy_raw_id,), - ).fetchone() == (native_id, 1, "byte_proven", "applied") - lines = [json.loads(line) for line in receipt.read_text().splitlines()] - assert [line["state"] for line in lines] == ["planned", "applied"] - assert lines[-1]["transaction_protocol"] == "rollback-superjournal-v1" - reapplied = repair_byte_proven_browser_capture_null_native_ids( - _config(tmp_path), [raw_id], apply=True, receipt_path=receipt, proof_digest=dry_run.proof_digest - ) - assert reapplied.repaired_count == 0 - assert reapplied.already_repaired_count == 1 - + preview = inspect_raw_authority_frontier(_config(tmp_path)) + selected = next(item for item in preview.items if item.raw_id == raw_id) + strategy = cast(dict[str, object], selected.strategy_witness["item"]) -@pytest.mark.parametrize("case", ("semantic-2af730ea", "semantic-27527c15")) -def test_byte_proven_browser_rekey_preserves_exact_semantic_witness(tmp_path: Path, case: str) -> None: - raw_id = _seed_byte_proven_browser_head_without_native_id(tmp_path) - semantic_raw_id = _seed_semantic_canonical_head(tmp_path, raw_id) - sibling_raw_id = _seed_semantic_superseded_sibling(tmp_path, semantic_raw_id) - historical_before = _rows(tmp_path, "source", "raw_sessions", "raw_id IN (?, ?)", (semantic_raw_id, sibling_raw_id)) - - dry_run = repair_byte_proven_browser_capture_null_native_ids(_config(tmp_path), [raw_id]) - - assert dry_run.eligible_count == 1, dry_run.items[0].reason - assert dry_run.items[0].semantic_canonical_raw_id == semantic_raw_id - assert dry_run.items[0].semantic_historical_raw_ids == (sibling_raw_id,) - receipt = tmp_path / f"byte-rekey-semantic-{case}.jsonl" - applied = repair_byte_proven_browser_capture_null_native_ids( - _config(tmp_path), [raw_id], apply=True, receipt_path=receipt, proof_digest=dry_run.proof_digest - ) + assert selected.state is RawAuthorityFrontierState.SAFELY_REKEYABLE + assert selected.actuator is RawAuthorityActuator.COPY_FORWARD_ORIGIN + assert strategy["legacy_null_native_id"] is (authority == "quarantined") + assert strategy["byte_proven_null_native_id_rekey"] is (authority == "byte_proven") - assert applied.repaired_count == 1 - assert ( - _rows(tmp_path, "source", "raw_sessions", "raw_id IN (?, ?)", (semantic_raw_id, sibling_raw_id)) - == historical_before + report = apply_raw_authority_frontier( + _config(tmp_path), + preview_census_id=preview.census_id, + selected_plan_ids=(selected.plan_id,), ) - assert applied.items[0].semantic_canonical_raw_id == semantic_raw_id - assert applied.items[0].semantic_historical_raw_ids == (sibling_raw_id,) - - -@pytest.mark.parametrize( - "conflict", - ("canonical_hash_conflict", "superseded_equivalent_membership", "canonical_byte_head", "current_reparse_drift"), -) -def test_byte_proven_browser_rekey_fails_closed_for_observed_conflicts(tmp_path: Path, conflict: str) -> None: - raw_id = _seed_byte_proven_browser_head_without_native_id(tmp_path) - if conflict == "canonical_hash_conflict": - semantic_raw_id = _seed_semantic_canonical_head(tmp_path, raw_id) - with sqlite3.connect(tmp_path / "index.db") as index: - index.execute( - "UPDATE raw_revision_heads SET accepted_content_hash = x'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF' " - "WHERE accepted_raw_id = ?", - (semantic_raw_id,), - ) - elif conflict == "superseded_equivalent_membership": - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute( - """ - INSERT INTO raw_session_memberships ( - raw_id, logical_source_key, provider_session_id, source_revision, - normalized_content_hash, message_count, acquisition_generation, - revision_authority, decision, decided_at_ms - ) SELECT raw_id, logical_source_key, 'browser-origin-one', source_revision, - x'0000000000000000000000000000000000000000000000000000000000000000', 1, 1, - 'quarantined', 'superseded_equivalent', 3 - FROM raw_sessions WHERE raw_id = ? - """, - (raw_id,), - ) - elif conflict == "canonical_byte_head": - _seed_equivalent_canonical_head(tmp_path, raw_id) - else: - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute("UPDATE raw_sessions SET source_revision = ? WHERE raw_id = ?", ("0" * 64, raw_id)) - - report = repair_byte_proven_browser_capture_null_native_ids(_config(tmp_path), [raw_id]) - - assert report.ineligible_count == 1 - assert report.items[0].status == "ineligible" - - -def test_byte_proven_browser_rekey_refuses_stale_proof_and_rolls_back( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - import polylogue.storage.repair as repair_module - - raw_id = _seed_byte_proven_browser_head_without_native_id(tmp_path) - dry_run = repair_byte_proven_browser_capture_null_native_ids(_config(tmp_path), [raw_id]) - with sqlite3.connect(tmp_path / "index.db") as index: - index.execute( - "UPDATE raw_revision_heads SET accepted_frontier = accepted_frontier + 1 WHERE accepted_raw_id = ?", - (raw_id,), - ) - with pytest.raises(RuntimeError, match="one or more targets are ineligible"): - repair_byte_proven_browser_capture_null_native_ids( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=tmp_path / "stale.jsonl", - proof_digest=dry_run.proof_digest, - ) - - raw_id = _seed_byte_proven_browser_head_without_native_id(tmp_path / "rollback") - dry_run = repair_byte_proven_browser_capture_null_native_ids(_config(tmp_path / "rollback"), [raw_id]) - before = { - (tier, table): _rows(tmp_path / "rollback", tier, table, "1 = 1", ()) - for tier, tables in { - "source": ("raw_sessions", "blob_refs", "raw_session_memberships", "raw_membership_census"), - "index": ("sessions", "raw_revision_heads", "raw_revision_applications"), - }.items() - for table in tables - } - - def fail_after_source_stage(conn: sqlite3.Connection, item: object) -> None: - raise RuntimeError("injected byte rekey index failure") - - monkeypatch.setattr(repair_module, "_apply_browser_origin_repair_item", fail_after_source_stage) - receipt = tmp_path / "rollback" / "rollback.jsonl" - with pytest.raises(RuntimeError, match="injected byte rekey index failure"): - repair_byte_proven_browser_capture_null_native_ids( - _config(tmp_path / "rollback"), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - assert [json.loads(line)["state"] for line in receipt.read_text().splitlines()] == ["planned"] - for (tier, table), rows in before.items(): - assert _rows(tmp_path / "rollback", tier, table, "1 = 1", ()) == rows - -# --- polylogue-lkrc.3: evidence packets + durable blockers for unresolved conflicts --- + assert report.executed_plan_count == 1 + assert report.retryable_plan_count == 0 def test_inspect_conflicts_reports_resolved_when_actuator_would_succeed(tmp_path: Path) -> None: @@ -2248,6 +700,80 @@ def test_inspect_conflicts_semantic_hash_divergence_evidence(tmp_path: Path) -> assert item.evidence_digest is not None +def test_unified_frontier_conflict_requires_typed_judgment_then_resumes_same_evidence(tmp_path: Path) -> None: + mismatched_raw_id = _seed_mismatched_browser_head(tmp_path) + canonical_raw_id = _seed_diverging_canonical_byte_head(tmp_path, mismatched_raw_id) + + census = inspect_raw_authority_frontier(_config(tmp_path)) + + conflict = next(item for item in census.items if item.raw_id == mismatched_raw_id) + assert conflict.state is RawAuthorityFrontierState.CONFLICTING_AUTHORITY_NEEDS_JUDGMENT + assert conflict.actuator is RawAuthorityActuator.REQUEST_JUDGMENT + assert conflict.executable is False + with sqlite3.connect(tmp_path / "source.db") as source: + blocker_id, observed_json = source.execute( + "SELECT blocker_id, observed_json FROM raw_authority_blockers WHERE plan_id = ? AND resolved_at_ms IS NULL", + (conflict.plan_id,), + ).fetchone() + assertion_id = json.loads(observed_json)["judgment_assertion_id"] + with sqlite3.connect(tmp_path / "user.db") as user: + assert user.execute("SELECT status FROM assertions WHERE assertion_id = ?", (assertion_id,)).fetchone() == ( + AssertionStatus.CANDIDATE.value, + ) + with pytest.raises(RuntimeError, match="explicitly accepted"): + resolve_raw_authority_blocker( + tmp_path, + blocker_id, + resolution="retain both authorities pending a future evidence change", + assertion_id=assertion_id, + ) + with sqlite3.connect(tmp_path / "user.db") as user, user: + assert mark_assertion_status(user, assertion_id, AssertionStatus.ACCEPTED) + with pytest.raises(RuntimeError, match="disposition=retain_canonical_authority"): + resolve_raw_authority_blocker( + tmp_path, + blocker_id, + resolution="retain the typed canonical authority", + assertion_id=assertion_id, + ) + resolved = resolve_raw_authority_blocker( + tmp_path, + blocker_id, + resolution="retain the typed canonical authority", + assertion_id=assertion_id, + judgment_disposition="retain_canonical_authority", + ) + assert resolved["operator_assertion_id"] == assertion_id + assert resolved["judgment_disposition"] == "retain_canonical_authority" + + repeated = inspect_raw_authority_frontier(_config(tmp_path)) + successor = next(item for item in repeated.items if item.raw_id == mismatched_raw_id) + assert successor.plan_id != conflict.plan_id + assert successor.state is RawAuthorityFrontierState.SAFELY_REKEYABLE + assert successor.actuator is RawAuthorityActuator.RESOLVE_CONFLICT + applied = apply_raw_authority_frontier( + _config(tmp_path), + preview_census_id=repeated.census_id, + selected_plan_ids=(successor.plan_id,), + ) + assert applied.executed_plan_count == 1 + assert applied.retryable_plan_count == 0 + postflight = inspect_raw_authority_frontier(_config(tmp_path)) + assert set(postflight.state_counts) <= { + RawAuthorityFrontierState.PROVEN_CURRENT.value, + RawAuthorityFrontierState.SUPERSEDED.value, + } + with sqlite3.connect(tmp_path / "index.db") as index: + assert index.execute( + "SELECT raw_id FROM sessions WHERE session_id = 'chatgpt-export:browser-origin-one'" + ).fetchone() == (canonical_raw_id,) + with sqlite3.connect(tmp_path / "source.db") as source: + assert source.execute( + "SELECT COUNT(*) FROM raw_authority_blockers WHERE plan_id = ? AND resolved_at_ms IS NULL", + (conflict.plan_id,), + ).fetchone() == (0,) + + def test_inspect_conflicts_membership_precondition_evidence(tmp_path: Path) -> None: raw_id = _seed_byte_proven_browser_head_without_native_id(tmp_path) with sqlite3.connect(tmp_path / "source.db") as source: @@ -2274,6 +800,19 @@ def test_inspect_conflicts_membership_precondition_evidence(tmp_path: Path) -> N assert "membership row" in item.divergence_note assert "superseded_equivalent" in item.divergence_note + census = inspect_raw_authority_frontier(_config(tmp_path)) + unresolved = next(frontier_item for frontier_item in census.items if frontier_item.raw_id == raw_id) + assert unresolved.state is RawAuthorityFrontierState.UNRESOLVED_PROVENANCE + assert unresolved.actuator is RawAuthorityActuator.NONE + assert unresolved.executable is False + with sqlite3.connect(tmp_path / "source.db") as source: + observed = source.execute( + "SELECT observed_json FROM raw_authority_blockers WHERE plan_id = ? AND resolved_at_ms IS NULL", + (unresolved.plan_id,), + ).fetchone() + assert observed is not None + assert "judgment_assertion_id" not in json.loads(str(observed[0])) + def test_inspect_conflicts_matching_hash_is_membership_shaped_not_a_divergence(tmp_path: Path) -> None: """Equal content hashes mean the actuator's block is precondition-shaped, not authority conflict.""" diff --git a/tests/unit/storage/test_duplicate_raw_identity_repair.py b/tests/unit/storage/test_duplicate_raw_identity_repair.py index 68ee585a55..09d8f907f2 100644 --- a/tests/unit/storage/test_duplicate_raw_identity_repair.py +++ b/tests/unit/storage/test_duplicate_raw_identity_repair.py @@ -1,16 +1,13 @@ """Tests for polylogue-t0dy: reconcile the pre-#2729 duplicate-raw scheme. -PR #2729 aligned the one-shot importer and the live daemon watcher on one -deterministic raw-id scheme (no ``native_id``) so *new* ingests of a grouped/ -split-session file converge on one raw row. It explicitly does not -retroactively repair pairs that already duplicated under the OLD, -native_id-inclusive scheme before that fix landed. ``repair_duplicate_raw_identity`` -is the typed, receipted, CAS-gated actuator that performs that one-time -reconciliation without hand-writing a raw UPDATE. +PR #2729 aligned the one-shot importer and live watcher on one deterministic +raw-id scheme. These tests prove the shared raw-authority reconciler discovers, +applies, and crash-recovers the resulting historical duplicate-alias state. """ from __future__ import annotations +import hashlib import json import sqlite3 from contextlib import closing @@ -23,7 +20,19 @@ from polylogue.core.enums import Provider, Role from polylogue.pipeline.ids import session_content_hash from polylogue.sources.parsers.base_models import ParsedMessage, ParsedSession -from polylogue.storage.repair import repair_duplicate_raw_identity +from polylogue.storage.archive_readiness import raw_materialization_readiness_snapshot, raw_materialization_ready +from polylogue.storage.raw_authority import ( + finalize_raw_authority_census, + read_raw_authority_detail, + record_raw_replay_outcome, +) +from polylogue.storage.raw_reconciler import ( + RawAuthorityActuator, + RawAuthorityFrontierState, + apply_raw_authority_frontier, + inspect_raw_authority_frontier, + recover_interrupted_raw_authority_frontier, +) from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root from polylogue.storage.sqlite.archive_tiers.revision_application import ( @@ -137,312 +146,249 @@ def _seed_duplicate_raw_pair(root: Path, *, legacy_native_id: str = "legacy-nati decided_at_ms=1, ) archive.commit() - return stale_raw_id, canonical_raw_id, session_id, logical_source_key + return stale_raw_id, canonical_raw_id, session_id, logical_source_key -def _rows(root: Path, tier: str, table: str, where: str, params: tuple[object, ...]) -> list[tuple[object, ...]]: - with closing(sqlite3.connect(root / f"{tier}.db")) as conn: - return sorted(conn.execute(f"SELECT * FROM {table} WHERE {where}", params).fetchall()) - +def test_unified_frontier_census_plans_duplicate_alias_with_stable_evidence(tmp_path: Path) -> None: + stale_raw_id, canonical_raw_id, _session_id, _logical_key = _seed_duplicate_raw_pair(tmp_path) -def test_dry_run_proves_eligible_pair_without_mutating(tmp_path: Path) -> None: - stale_raw_id, canonical_raw_id, session_id, _key = _seed_duplicate_raw_pair(tmp_path) - before = { - (tier, table): _rows(tmp_path, tier, table, "1 = 1", ()) - for tier, tables in { - "source": ("raw_sessions", "blob_refs"), - "index": ("sessions", "raw_revision_heads", "raw_revision_applications"), - }.items() - for table in tables - } - - report = repair_duplicate_raw_identity(_config(tmp_path), [(stale_raw_id, canonical_raw_id)]) - - assert report.mode == "dry-run" - assert report.requested_count == 1 - assert report.eligible_count == 1 - assert report.ineligible_count == 0 - assert report.repaired_count == 0 - item = report.items[0] - assert item.status == "eligible" - assert item.session_id == session_id - assert item.logical_source_key == "codex:carryover-session" - assert item.proof_digest is not None - for key, rows in before.items(): - assert _rows(tmp_path, *key, "1 = 1", ()) == rows - - -def test_apply_repoints_head_and_session_preserving_stale_raw(tmp_path: Path) -> None: - stale_raw_id, canonical_raw_id, session_id, key = _seed_duplicate_raw_pair(tmp_path) - stale_before = _rows(tmp_path, "source", "raw_sessions", "raw_id = ?", (stale_raw_id,)) - - dry_run = repair_duplicate_raw_identity(_config(tmp_path), [(stale_raw_id, canonical_raw_id)]) - receipt = tmp_path / "t0dy-repair.json" - applied = repair_duplicate_raw_identity( - _config(tmp_path), - [(stale_raw_id, canonical_raw_id)], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) + first = inspect_raw_authority_frontier(_config(tmp_path)) + second = inspect_raw_authority_frontier(_config(tmp_path)) - assert applied.mode == "apply" - assert applied.repaired_count == 1 - assert applied.items[0].repaired is True - assert receipt.exists() + duplicate = next(item for item in first.items if item.raw_id == stale_raw_id) + duplicate_again = next(item for item in second.items if item.raw_id == stale_raw_id) + assert duplicate.state is RawAuthorityFrontierState.DUPLICATE_ALIAS + assert duplicate.actuator is RawAuthorityActuator.FOLD_DUPLICATE_ALIAS + assert duplicate.input_raw_ids == tuple(sorted((stale_raw_id, canonical_raw_id))) + assert duplicate.plan_id == duplicate_again.plan_id + assert duplicate.evidence_digest == duplicate_again.evidence_digest + assert duplicate.evidence_ref is not None + assert first.state_counts[RawAuthorityFrontierState.DUPLICATE_ALIAS.value] == 1 + assert first.executable_plan_count == 1 - # The stale raw's own row is byte-for-byte unchanged -- durable raw - # evidence is never mutated or deleted, only its authority. - assert _rows(tmp_path, "source", "raw_sessions", "raw_id = ?", (stale_raw_id,)) == stale_before - - with closing(sqlite3.connect(tmp_path / "index.db")) as index_conn: - head = index_conn.execute( - "SELECT accepted_raw_id FROM raw_revision_heads WHERE logical_source_key = ?", (key,) - ).fetchone() - assert head[0] == canonical_raw_id - session_raw = index_conn.execute("SELECT raw_id FROM sessions WHERE session_id = ?", (session_id,)).fetchone() - assert session_raw[0] == canonical_raw_id - # The stale raw carries BOTH its original ``selected_baseline`` receipt - # (from before repair) and the new ``superseded`` receipt (from - # repair) -- immutable application receipts are append-only, never - # overwritten -- so assert presence of the new one specifically - # rather than assuming there is only one row. - stale_decisions = { - row[0] - for row in index_conn.execute( - "SELECT decision FROM raw_revision_applications WHERE raw_id = ? AND logical_source_key = ?", - (stale_raw_id, key), - ).fetchall() - } - assert stale_decisions == {"selected_baseline", "superseded"} - stale_superseded_target = index_conn.execute( - "SELECT accepted_raw_id FROM raw_revision_applications " - "WHERE raw_id = ? AND logical_source_key = ? AND decision = 'superseded'", - (stale_raw_id, key), - ).fetchone() - assert stale_superseded_target[0] == canonical_raw_id - canonical_decision = index_conn.execute( - "SELECT decision, accepted_raw_id FROM raw_revision_applications WHERE raw_id = ? AND logical_source_key = ?", - (canonical_raw_id, key), + with sqlite3.connect(tmp_path / "source.db") as conn: + persisted = conn.execute( + """ + SELECT p.input_raw_ids_json, p.authority_witness_json, + p.source_preconditions_json, p.index_preconditions_json + FROM raw_authority_census_plans AS cp + JOIN raw_authority_plans AS p ON p.plan_id = cp.plan_id + WHERE cp.census_id = ? AND p.plan_id = ? + """, + (first.census_id, duplicate.plan_id), ).fetchone() - assert canonical_decision == ("selected_baseline", canonical_raw_id) + assert persisted is not None + assert json.loads(persisted[0]) == sorted((stale_raw_id, canonical_raw_id)) + assert json.loads(persisted[1])["actuator"] == RawAuthorityActuator.FOLD_DUPLICATE_ALIAS.value + assert json.loads(persisted[2])["blob_hash"] + assert json.loads(persisted[3])["accepted_content_hash"] + + +def test_unified_frontier_census_prioritizes_missing_bytes_over_safe_actuation(tmp_path: Path) -> None: + stale_raw_id, _canonical_raw_id, _session_id, _logical_key = _seed_duplicate_raw_pair(tmp_path) + with sqlite3.connect(tmp_path / "source.db") as conn: + blob_hash = bytes( + conn.execute("SELECT blob_hash FROM raw_sessions WHERE raw_id = ?", (stale_raw_id,)).fetchone()[0] + ) + blob_path = tmp_path / "blob" / blob_hash.hex()[:2] / blob_hash.hex()[2:] + blob_path.unlink() + census = inspect_raw_authority_frontier(_config(tmp_path)) -def test_reapply_after_success_is_idempotent_already_repaired(tmp_path: Path) -> None: - stale_raw_id, canonical_raw_id, _session_id, _key = _seed_duplicate_raw_pair(tmp_path) - dry_run = repair_duplicate_raw_identity(_config(tmp_path), [(stale_raw_id, canonical_raw_id)]) - repair_duplicate_raw_identity( - _config(tmp_path), - [(stale_raw_id, canonical_raw_id)], - apply=True, - receipt_path=tmp_path / "first.json", - proof_digest=dry_run.proof_digest, + missing = next(item for item in census.items if item.raw_id == stale_raw_id) + assert missing.state is RawAuthorityFrontierState.MISSING_BYTES_REACQUIRE + assert missing.actuator is RawAuthorityActuator.REACQUIRE + assert missing.executable is False + with sqlite3.connect(tmp_path / "source.db") as conn: + obligation = conn.execute( + """ + SELECT b.reason, b.resolved_at_ms, p.authority_witness_json + FROM raw_authority_blockers AS b + JOIN raw_authority_plans AS p ON p.plan_id = b.plan_id + WHERE b.plan_id = ? + """, + (missing.plan_id,), + ).fetchone() + assert obligation is not None + assert obligation[1] is None + assert json.loads(obligation[2])["state"] == RawAuthorityFrontierState.MISSING_BYTES_REACQUIRE.value + readiness = raw_materialization_readiness_snapshot(tmp_path) + assert readiness["raw_authority_frontier_blocking_count"] == 1 + assert raw_materialization_ready(readiness) is False + refs = readiness["raw_authority_frontier_remediation_refs"] + assert isinstance(refs, list) and refs[0]["plan_id"] == missing.plan_id + detail = read_raw_authority_detail(tmp_path, str(refs[0]["detail_query_handle"])) + assert missing.plan_id in str(detail["chunk"]) + + blob_path.parent.mkdir(parents=True, exist_ok=True) + blob_path.write_bytes(b"wrong bytes at the expected content-addressed path") + still_missing = inspect_raw_authority_frontier(_config(tmp_path)) + wrong_bytes = next(item for item in still_missing.items if item.raw_id == stale_raw_id) + assert wrong_bytes.state is RawAuthorityFrontierState.MISSING_BYTES_REACQUIRE + assert "do not prove" in wrong_bytes.reason + + reacquired = json.dumps({"marker": "duplicate-raw-fixture", "legacy_native_id": "legacy-native-id-1"}).encode() + assert hashlib.sha256(reacquired).digest() == blob_hash + blob_path.write_bytes(reacquired) + advanced = inspect_raw_authority_frontier(_config(tmp_path)) + assert ( + next(item for item in advanced.items if item.raw_id == stale_raw_id).state + is RawAuthorityFrontierState.DUPLICATE_ALIAS ) + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT COUNT(*) FROM raw_authority_blockers WHERE plan_id = ? AND resolved_at_ms IS NULL", + (missing.plan_id,), + ).fetchone() == (0,) + + +def test_unified_frontier_first_census_rejects_replaced_blob_bytes(tmp_path: Path) -> None: + stale_raw_id, _canonical_raw_id, _session_id, _logical_key = _seed_duplicate_raw_pair(tmp_path) + with sqlite3.connect(tmp_path / "source.db") as conn: + blob_hash = bytes( + conn.execute("SELECT blob_hash FROM raw_sessions WHERE raw_id = ?", (stale_raw_id,)).fetchone()[0] + ) + blob_path = tmp_path / "blob" / blob_hash.hex()[:2] / blob_hash.hex()[2:] + blob_path.write_bytes(b"replacement bytes present before the first census") - again = repair_duplicate_raw_identity(_config(tmp_path), [(stale_raw_id, canonical_raw_id)]) + census = inspect_raw_authority_frontier(_config(tmp_path)) - assert again.mode == "dry-run" - assert again.already_repaired_count == 1 - assert again.eligible_count == 0 - assert again.ineligible_count == 0 - assert again.items[0].status == "already_repaired" + replaced = next(item for item in census.items if item.raw_id == stale_raw_id) + assert replaced.state is RawAuthorityFrontierState.MISSING_BYTES_REACQUIRE + assert replaced.actuator is RawAuthorityActuator.REACQUIRE -@pytest.mark.parametrize( - "mutation", - ("canonical_already_accepted", "stale_not_accepted", "content_differs", "same_scheme_both_null"), -) -def test_fails_closed_for_ineligible_shapes(tmp_path: Path, mutation: str) -> None: - stale_raw_id, canonical_raw_id, session_id, key = _seed_duplicate_raw_pair(tmp_path) - if mutation == "canonical_already_accepted": - with closing(sqlite3.connect(tmp_path / "index.db")) as index_conn, index_conn: - index_conn.execute( - "INSERT INTO raw_revision_heads (logical_source_key, session_id, accepted_raw_id, " - "accepted_source_revision, accepted_content_hash, accepted_frontier_kind, accepted_frontier, " - "acquisition_generation, decided_at_ms) " - "SELECT 'codex:other-session', session_id, ?, accepted_source_revision, accepted_content_hash, " - "accepted_frontier_kind, accepted_frontier, acquisition_generation, decided_at_ms " - "FROM raw_revision_heads WHERE logical_source_key = ?", - (canonical_raw_id, key), - ) - elif mutation == "stale_not_accepted": - with closing(sqlite3.connect(tmp_path / "index.db")) as index_conn, index_conn: - index_conn.execute( - "UPDATE raw_revision_heads SET accepted_raw_id = ? WHERE logical_source_key = ?", - (canonical_raw_id, key), - ) - elif mutation == "content_differs": - with closing(sqlite3.connect(tmp_path / "source.db")) as source_conn, source_conn: - source_conn.execute( - "UPDATE raw_sessions SET blob_size = blob_size + 1 WHERE raw_id = ?", (canonical_raw_id,) - ) - else: - with closing(sqlite3.connect(tmp_path / "source.db")) as source_conn, source_conn: - source_conn.execute("UPDATE raw_sessions SET native_id = NULL WHERE raw_id = ?", (stale_raw_id,)) - - report = repair_duplicate_raw_identity(_config(tmp_path), [(stale_raw_id, canonical_raw_id)]) - - assert report.ineligible_count == 1 - assert report.items[0].status == "ineligible" - with pytest.raises(RuntimeError, match="ineligible"): - repair_duplicate_raw_identity( +def test_unified_frontier_apply_obeys_offline_daemon_guard( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + stale_raw_id, _canonical_raw_id, _session_id, _logical_key = _seed_duplicate_raw_pair(tmp_path) + preview = inspect_raw_authority_frontier(_config(tmp_path)) + selected = next(item for item in preview.items if item.raw_id == stale_raw_id) + monkeypatch.setattr( + "polylogue.maintenance.offline_guard.offline_maintenance_block_reason", + lambda *_args, **_kwargs: "daemon owns the archive write lease", + ) + + with pytest.raises(RuntimeError, match="daemon owns"): + apply_raw_authority_frontier( _config(tmp_path), - [(stale_raw_id, canonical_raw_id)], - apply=True, - receipt_path=tmp_path / "should-not-write.json", - proof_digest=report.proof_digest, + preview_census_id=preview.census_id, + selected_plan_ids=(selected.plan_id,), ) -def test_rejects_duplicate_pairs_and_malformed_ids(tmp_path: Path) -> None: - stale_raw_id, canonical_raw_id, _session_id, _key = _seed_duplicate_raw_pair(tmp_path) - with pytest.raises(ValueError, match="duplicate"): - repair_duplicate_raw_identity( - _config(tmp_path), [(stale_raw_id, canonical_raw_id), (stale_raw_id, canonical_raw_id)] - ) - with pytest.raises(ValueError, match="lowercase SHA-256"): - repair_duplicate_raw_identity(_config(tmp_path), [("not-a-raw-id", canonical_raw_id)]) - with pytest.raises(ValueError, match="1..100 entries"): - repair_duplicate_raw_identity(_config(tmp_path), []) - - -def test_apply_refuses_stale_proof_digest(tmp_path: Path) -> None: - stale_raw_id, canonical_raw_id, _session_id, key = _seed_duplicate_raw_pair(tmp_path) - dry_run = repair_duplicate_raw_identity(_config(tmp_path), [(stale_raw_id, canonical_raw_id)]) - with closing(sqlite3.connect(tmp_path / "index.db")) as index_conn, index_conn: - index_conn.execute( - "UPDATE raw_revision_heads SET decided_at_ms = decided_at_ms + 1 WHERE logical_source_key = ?", (key,) +def test_unified_frontier_apply_drives_duplicate_strategy_and_postflight(tmp_path: Path) -> None: + stale_raw_id, canonical_raw_id, session_id, logical_key = _seed_duplicate_raw_pair(tmp_path) + preview = inspect_raw_authority_frontier(_config(tmp_path)) + selected = next(item for item in preview.items if item.raw_id == stale_raw_id) + + report = apply_raw_authority_frontier( + _config(tmp_path), + preview_census_id=preview.census_id, + selected_plan_ids=(selected.plan_id,), + ) + + assert report.success is True + assert report.selected_plan_count == report.executed_plan_count == 1 + assert report.retryable_plan_count == 0 + assert len(report.outcome_refs) == 1 + with sqlite3.connect(tmp_path / "index.db") as conn: + assert conn.execute( + "SELECT accepted_raw_id FROM raw_revision_heads WHERE logical_source_key = ?", + (logical_key,), + ).fetchone() == (canonical_raw_id,) + assert conn.execute("SELECT raw_id FROM sessions WHERE session_id = ?", (session_id,)).fetchone() == ( + canonical_raw_id, ) + with sqlite3.connect(tmp_path / "source.db") as conn: + row = conn.execute( + "SELECT lifecycle_status FROM raw_authority_censuses WHERE census_id = ?", + (report.census_id,), + ).fetchone() + assert row == ("completed",) + assert conn.execute("SELECT COUNT(*) FROM raw_sessions WHERE raw_id = ?", (stale_raw_id,)).fetchone() == (1,) + postflight = inspect_raw_authority_frontier(_config(tmp_path)) + assert any( + item.raw_id == stale_raw_id and item.state is RawAuthorityFrontierState.SUPERSEDED for item in postflight.items + ) + + +def test_unified_frontier_recovers_crash_after_strategy_commit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + stale_raw_id, canonical_raw_id, _session_id, logical_key = _seed_duplicate_raw_pair(tmp_path) + preview = inspect_raw_authority_frontier(_config(tmp_path)) + selected = next(item for item in preview.items if item.raw_id == stale_raw_id) - with pytest.raises(RuntimeError, match="proof digest does not match"): - repair_duplicate_raw_identity( + def crash_before_outcome(*_args: object, **_kwargs: object) -> None: + raise RuntimeError("injected crash after strategy commit") + + monkeypatch.setattr("polylogue.storage.raw_reconciler.record_raw_replay_outcome", crash_before_outcome) + with pytest.raises(RuntimeError, match="injected crash"): + apply_raw_authority_frontier( _config(tmp_path), - [(stale_raw_id, canonical_raw_id)], - apply=True, - receipt_path=tmp_path / "stale-digest.json", - proof_digest=dry_run.proof_digest, + preview_census_id=preview.census_id, + selected_plan_ids=(selected.plan_id,), ) + monkeypatch.setattr("polylogue.storage.raw_reconciler.record_raw_replay_outcome", record_raw_replay_outcome) + recovered = recover_interrupted_raw_authority_frontier(_config(tmp_path)) -def test_apply_serializes_one_receipt_inode_against_a_concurrent_apply(tmp_path: Path) -> None: - """A second ``--apply`` against the same receipt path must fail closed. + assert recovered == (selected.plan_id,) + with sqlite3.connect(tmp_path / "index.db") as conn: + assert conn.execute( + "SELECT accepted_raw_id FROM raw_revision_heads WHERE logical_source_key = ?", + (logical_key,), + ).fetchone() == (canonical_raw_id,) + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT lifecycle_status FROM raw_authority_censuses WHERE mode = 'apply' ORDER BY sequence_no DESC LIMIT 1" + ).fetchone() == ("interrupted",) - Exercises ``_lock_duplicate_raw_identity_repair_receipt`` directly: hold - its flock open (as a genuinely concurrent apply would), then prove the - real ``repair_duplicate_raw_identity`` entrypoint refuses to proceed and - never mutates the durable authority underneath the held lock. Deleting - the flock acquisition from the receipt lock (regressing to the old bare - ``receipt_path.exists()`` TOCTOU check) makes this test fail: the second - call would instead race straight into the transaction. - """ - stale_raw_id, canonical_raw_id, _session_id, key = _seed_duplicate_raw_pair(tmp_path) - dry_run = repair_duplicate_raw_identity(_config(tmp_path), [(stale_raw_id, canonical_raw_id)]) - receipt_path = tmp_path / "locked.jsonl" - from polylogue.storage import repair as repair_module - - locked = repair_module._lock_duplicate_raw_identity_repair_receipt(receipt_path, list(dry_run.items)) - try: - with pytest.raises(RuntimeError, match="already locked"): - repair_duplicate_raw_identity( - _config(tmp_path), - [(stale_raw_id, canonical_raw_id)], - apply=True, - receipt_path=receipt_path, - proof_digest=dry_run.proof_digest, - ) - with closing(sqlite3.connect(tmp_path / "index.db")) as index_conn: - head = index_conn.execute( - "SELECT accepted_raw_id FROM raw_revision_heads WHERE logical_source_key = ?", (key,) - ).fetchone() - assert head[0] == stale_raw_id - finally: - locked.close() - - -def test_apply_resumes_planned_receipt_after_a_crash_before_the_terminal_append( + +def test_unified_frontier_recovers_crash_after_outcome_before_postflight( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - """A crash between commit and the terminal receipt append must be resumable and auditable. - - Injects a failure into ``_finish_duplicate_raw_identity_repair_receipt`` - (the real production function that appends the fsynced ``applied`` - record) after the index-tier transaction has already committed. The - receipt on disk must show only the ``planned`` record -- proving the - planned phase is durably written *before* mutation, unlike the old - single unlocked ``write_text`` -- and re-invoking apply against the SAME - receipt path must resume to a terminal ``applied`` record without - re-mutating anything (idempotent recovery), once the operator re-proves - current authority with a fresh dry-run digest (the accepted head's own - ``decided_at_ms`` legitimately changed under repair, so reusing the - original pre-repair digest is correctly refused by the top-level CAS - gate -- this mirrors exactly how an operator would recover in practice). - Removing the planned-phase write or the crash-safe resume path makes - this test fail. - """ - stale_raw_id, canonical_raw_id, _session_id, _key = _seed_duplicate_raw_pair(tmp_path) - dry_run = repair_duplicate_raw_identity(_config(tmp_path), [(stale_raw_id, canonical_raw_id)]) - receipt = tmp_path / "planned-resume.jsonl" - from polylogue.storage import repair as repair_module + stale_raw_id, _canonical_raw_id, _session_id, _logical_key = _seed_duplicate_raw_pair(tmp_path) + preview = inspect_raw_authority_frontier(_config(tmp_path)) + selected = next(item for item in preview.items if item.raw_id == stale_raw_id) - original_finish = repair_module._finish_duplicate_raw_identity_repair_receipt - monkeypatch.setattr( - repair_module, - "_finish_duplicate_raw_identity_repair_receipt", - lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("injected terminal append crash")), - ) - with pytest.raises(RuntimeError, match="terminal append crash"): - repair_duplicate_raw_identity( + def crash_before_postflight(*_args: object, **_kwargs: object) -> None: + raise RuntimeError("injected crash after durable outcome") + + monkeypatch.setattr("polylogue.storage.raw_reconciler.finalize_raw_authority_census", crash_before_postflight) + with pytest.raises(RuntimeError, match="injected crash"): + apply_raw_authority_frontier( _config(tmp_path), - [(stale_raw_id, canonical_raw_id)], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, + preview_census_id=preview.census_id, + selected_plan_ids=(selected.plan_id,), ) - assert [json.loads(line)["state"] for line in receipt.read_text().splitlines()] == ["planned"] - with closing(sqlite3.connect(tmp_path / "index.db")) as index_conn: - session_raw = index_conn.execute("SELECT raw_id FROM sessions WHERE session_id = ?", (_session_id,)).fetchone() - assert session_raw[0] == canonical_raw_id - - monkeypatch.setattr(repair_module, "_finish_duplicate_raw_identity_repair_receipt", original_finish) - post_crash_dry_run = repair_duplicate_raw_identity(_config(tmp_path), [(stale_raw_id, canonical_raw_id)]) - assert post_crash_dry_run.already_repaired_count == 1 - resumed = repair_duplicate_raw_identity( - _config(tmp_path), - [(stale_raw_id, canonical_raw_id)], - apply=True, - receipt_path=receipt, - proof_digest=post_crash_dry_run.proof_digest, + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + """ + SELECT lifecycle_status, outcome_recorded + FROM raw_authority_censuses AS c + JOIN raw_authority_census_plans AS cp USING (census_id) + WHERE c.mode = 'apply' ORDER BY c.sequence_no DESC LIMIT 1 + """ + ).fetchone() == ("planned", 1) + + monkeypatch.setattr( + "polylogue.storage.raw_reconciler.finalize_raw_authority_census", + finalize_raw_authority_census, ) - assert resumed.already_repaired_count == 1 - assert resumed.repaired_count == 0 - records = [json.loads(line) for line in receipt.read_text().splitlines()] - assert [record["state"] for record in records] == ["planned", "applied"] - assert records[1]["repaired_stale_raw_ids"] == [] - - -def test_receipt_path_must_not_be_a_symlink(tmp_path: Path) -> None: - stale_raw_id, canonical_raw_id, _session_id, _key = _seed_duplicate_raw_pair(tmp_path) - dry_run = repair_duplicate_raw_identity(_config(tmp_path), [(stale_raw_id, canonical_raw_id)]) - target = tmp_path / "outside-target.jsonl" - receipt = tmp_path / "receipt-symlink.jsonl" - receipt.symlink_to(target) - - with pytest.raises(RuntimeError, match="symbolic link"): - repair_duplicate_raw_identity( - _config(tmp_path), - [(stale_raw_id, canonical_raw_id)], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - assert not target.exists() + recovered = recover_interrupted_raw_authority_frontier(_config(tmp_path)) + assert recovered == () + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute( + "SELECT lifecycle_status FROM raw_authority_censuses WHERE mode = 'apply' ORDER BY sequence_no DESC LIMIT 1" + ).fetchone() == ("interrupted",) -def test_apply_requires_receipt_path(tmp_path: Path) -> None: - stale_raw_id, canonical_raw_id, _session_id, _key = _seed_duplicate_raw_pair(tmp_path) - dry_run = repair_duplicate_raw_identity(_config(tmp_path), [(stale_raw_id, canonical_raw_id)]) - with pytest.raises(ValueError, match="explicit operator repair receipt path"): - repair_duplicate_raw_identity( - _config(tmp_path), [(stale_raw_id, canonical_raw_id)], apply=True, proof_digest=dry_run.proof_digest - ) + +def _rows(root: Path, tier: str, table: str, where: str, params: tuple[object, ...]) -> list[tuple[object, ...]]: + with closing(sqlite3.connect(root / f"{tier}.db")) as conn: + return sorted(conn.execute(f"SELECT * FROM {table} WHERE {where}", params).fetchall()) diff --git a/tests/unit/storage/test_quarantined_accepted_raw_repair.py b/tests/unit/storage/test_quarantined_accepted_raw_repair.py index 5b8f178bc0..eb9a5b0b2b 100644 --- a/tests/unit/storage/test_quarantined_accepted_raw_repair.py +++ b/tests/unit/storage/test_quarantined_accepted_raw_repair.py @@ -2,10 +2,8 @@ import hashlib import json -import os import sqlite3 from pathlib import Path -from typing import Any import pytest @@ -15,8 +13,11 @@ from polylogue.pipeline.ids import session_content_hash from polylogue.sources.revision_backfill import _parse_one from polylogue.storage.blob_store import BlobStore -from polylogue.storage.index_generation import RebuildLease -from polylogue.storage.repair import repair_quarantined_accepted_raws +from polylogue.storage.raw_reconciler import ( + RawAuthorityFrontierState, + apply_raw_authority_frontier, + inspect_raw_authority_frontier, +) from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root from polylogue.storage.sqlite.archive_tiers.revision_application import ( @@ -191,7 +192,7 @@ def _retarget_fixture_raw_id(root: Path, old_raw_id: str, new_raw_id: str) -> No @pytest.mark.parametrize("typed_quarantined", [False, True]) -def test_quarantined_accepted_raw_repair_roundtrip_is_receipted_and_idempotent( +def test_unified_frontier_applies_quarantine_refinement_without_incident_receipt( tmp_path: Path, typed_quarantined: bool ) -> None: raw_id = _seed_invalid_head(tmp_path, typed_quarantined=typed_quarantined) @@ -202,849 +203,60 @@ def test_quarantined_accepted_raw_repair_roundtrip_is_receipted_and_idempotent( artifact_id, raw_id, origin, source_path, source_index, artifact_kind, support_status, classification_reason, parse_as_session, schema_eligible, malformed_jsonl_lines, first_observed_at_ms, last_observed_at_ms - ) VALUES ('artifact-witness', ?, 'chatgpt-export', 'repair-one.json', 0, + ) VALUES ('unified-artifact-witness', ?, 'chatgpt-export', 'repair-one.json', 0, 'session', 'supported_parseable', 'witness', 1, 1, 0, 1, 2) """, (raw_id,), ) - before = _logical_state(tmp_path, raw_id) - before_raw = _raw_session_row(tmp_path, raw_id) - - dry_run = repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]) - - assert dry_run.eligible_count == 1, dry_run.items[0].reason - assert dry_run.items[0].proof_digest - assert dry_run.items[0].application_decision_id - assert dry_run.items[0].origin == "chatgpt-export" - assert dry_run.items[0].source_index == 0 - assert dry_run.items[0].blob_hash == dry_run.items[0].accepted_source_revision - assert dry_run.items[0].blob_ref_hash == dry_run.items[0].blob_hash - assert dry_run.items[0].accepted_frontier_kind == "byte" - assert dry_run.items[0].accepted_frontier == dry_run.items[0].blob_size - assert dry_run.items[0].head_decided_at_ms == 2 - assert [artifact.artifact_id for artifact in dry_run.items[0].artifact_witnesses] == ["artifact-witness"] - assert dry_run.items[0].application_witness is not None - assert dry_run.items[0].application_witness.detail == "newest unique byte-proven full baseline" - assert _logical_state(tmp_path, raw_id) == before + preview = inspect_raw_authority_frontier(_config(tmp_path)) + selected = next(item for item in preview.items if item.raw_id == raw_id) - receipt_path = tmp_path / "recovery" / "quarantined-raw-repair.jsonl" - receipt_path.parent.mkdir() - applied = repair_quarantined_accepted_raws( + report = apply_raw_authority_frontier( _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt_path, - proof_digest=dry_run.proof_digest, + preview_census_id=preview.census_id, + selected_plan_ids=(selected.plan_id,), ) - assert applied.repaired_count == 1 - assert applied.items[0].status == "already_repaired" - records = [json.loads(line) for line in receipt_path.read_text().splitlines()] - assert [record["state"] for record in records] == ["planned", "applied"] - with sqlite3.connect(tmp_path / "source.db") as conn: - envelope = conn.execute( - """ - SELECT logical_source_key, revision_kind, source_revision, baseline_raw_id, - acquisition_generation, revision_authority - FROM raw_sessions WHERE raw_id = ? - """, + assert report.executed_plan_count == 1 + assert report.retryable_plan_count == 0 + with sqlite3.connect(tmp_path / "source.db") as source: + assert source.execute( + "SELECT revision_authority, baseline_raw_id FROM raw_sessions WHERE raw_id = ?", (raw_id,), - ).fetchone() - assert envelope == ( - "chatgpt:repair-one", - "full", - dry_run.items[0].accepted_source_revision, - raw_id, - 0, - "byte_proven", - ) - after = _logical_state(tmp_path, raw_id) - for key in before: - if key != "source.raw_sessions": - assert after[key] == before[key] - after_raw = _raw_session_row(tmp_path, raw_id) - expected_updates = { - "logical_source_key": "chatgpt:repair-one", - "revision_kind": "full", - "source_revision": dry_run.items[0].accepted_source_revision, - "baseline_raw_id": raw_id, - "acquisition_generation": 0, - "revision_authority": "byte_proven", - } - if typed_quarantined: - expected_updates = {"baseline_raw_id": raw_id, "revision_authority": "byte_proven"} - assert {key: after_raw[key] for key in expected_updates} == expected_updates - assert {key for key in before_raw if before_raw[key] != after_raw[key]} == set(expected_updates) - - reapplied = repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt_path, - proof_digest=dry_run.proof_digest, - ) - assert reapplied.repaired_count == 0 - assert receipt_path.read_text().count("\n") == 2 + ).fetchone() == ("byte_proven", raw_id) + assert not (tmp_path / "recovery").exists() @pytest.mark.parametrize( "mutation", - [ - "head_raw", - "missing_blob", - "blob_ref", - "artifact", - "frontier", - "session_hash", - "origin", - "capture_mode", - "application", - "receipt_time", - "typed_competitor", - "second_indexed_session", - "membership", - "envelope", - "multi_session", - ], + ["missing_blob", "blob_ref", "frontier", "session_hash", "application", "membership", "envelope"], ) -def test_quarantined_accepted_raw_repair_mutations_fail_closed(tmp_path: Path, mutation: str) -> None: - raw_id = _seed_invalid_head(tmp_path, multi_session=mutation == "multi_session") +def test_unified_quarantine_strategy_rejects_mutated_authority_witness(tmp_path: Path, mutation: str) -> None: + raw_id = _seed_invalid_head(tmp_path) with sqlite3.connect(tmp_path / "source.db") as source, sqlite3.connect(tmp_path / "index.db") as index: - if mutation == "head_raw": - index.execute("UPDATE raw_revision_heads SET accepted_raw_id = ?", ("0" * 64,)) - elif mutation == "missing_blob": - blob_hash = source.execute( - "SELECT hex(blob_hash) FROM raw_sessions WHERE raw_id = ?", (raw_id,) - ).fetchone()[0] - BlobStore(tmp_path / "blob").blob_path(str(blob_hash).lower()).unlink() + if mutation == "missing_blob": + blob_hash = str( + source.execute("SELECT hex(blob_hash) FROM raw_sessions WHERE raw_id = ?", (raw_id,)).fetchone()[0] + ) + BlobStore(tmp_path / "blob").blob_path(blob_hash.lower()).unlink() elif mutation == "blob_ref": source.execute("UPDATE blob_refs SET size_bytes = size_bytes + 1 WHERE ref_id = ?", (raw_id,)) - elif mutation == "artifact": - source.execute( - """ - INSERT INTO raw_artifacts ( - artifact_id, raw_id, origin, source_path, source_index, artifact_kind, - support_status, classification_reason, parse_as_session, schema_eligible, - malformed_jsonl_lines, first_observed_at_ms, last_observed_at_ms - ) VALUES ('artifact', ?, 'chatgpt-export', 'wrong.json', 0, 'session', - 'supported_parseable', 'test', 1, 1, 0, 1, 1) - """, - (raw_id,), - ) elif mutation == "frontier": index.execute("UPDATE raw_revision_heads SET accepted_frontier = accepted_frontier + 1") elif mutation == "session_hash": index.execute("UPDATE sessions SET content_hash = zeroblob(32)") - elif mutation == "origin": - source.execute("UPDATE raw_sessions SET origin = 'claude-ai-export' WHERE raw_id = ?", (raw_id,)) - elif mutation == "capture_mode": - source.execute( - "UPDATE raw_sessions SET origin = 'aistudio-drive', capture_mode = NULL WHERE raw_id = ?", (raw_id,) - ) elif mutation == "application": - index.execute( - """ - INSERT INTO raw_revision_applications ( - decision_id, raw_id, session_id, logical_source_key, source_revision, - acquisition_generation, decision, detail, decided_at_ms - ) SELECT 'competing', raw_id, session_id, logical_source_key, source_revision, - acquisition_generation, 'ambiguous', 'test', decided_at_ms + 1 - FROM raw_revision_applications LIMIT 1 - """ - ) - elif mutation == "receipt_time": - index.execute("UPDATE raw_revision_applications SET decided_at_ms = decided_at_ms + 1") - elif mutation == "typed_competitor": - source.execute( - """ - INSERT INTO raw_sessions ( - raw_id, origin, source_path, source_index, blob_hash, blob_size, acquired_at_ms, - logical_source_key, revision_kind, source_revision, baseline_raw_id, - acquisition_generation, revision_authority - ) SELECT ?, origin, source_path, source_index, blob_hash, blob_size, acquired_at_ms, - 'chatgpt:repair-one', 'full', ?, ?, 1, 'byte_proven' - FROM raw_sessions WHERE raw_id = ? - """, - ("1" * 64, "2" * 64, "1" * 64, raw_id), - ) - elif mutation == "second_indexed_session": - index.execute( - """ - INSERT INTO sessions (native_id, origin, raw_id, content_hash) - SELECT 'unexpected-second-session', origin, raw_id, content_hash - FROM sessions LIMIT 1 - """ - ) + index.execute("UPDATE raw_revision_applications SET accepted_raw_id = ?", ("1" * 64,)) elif mutation == "membership": - source.execute( - "UPDATE raw_membership_census SET status = 'failed', member_count = 0, detail = 'ambiguous' WHERE raw_id = ?", - (raw_id,), - ) + source.execute("UPDATE raw_membership_census SET status = 'failed', member_count = 0") elif mutation == "envelope": source.execute("UPDATE raw_sessions SET logical_source_key = 'partial' WHERE raw_id = ?", (raw_id,)) source.commit() index.commit() before = _logical_state(tmp_path, raw_id) - report = repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]) + census = inspect_raw_authority_frontier(_config(tmp_path)) + item = next(item for item in census.items if item.raw_id == raw_id) - assert report.ineligible_count == 1 + assert item.state is not RawAuthorityFrontierState.SAFELY_REKEYABLE assert _logical_state(tmp_path, raw_id) == before - - -def test_quarantined_accepted_raw_repair_preserves_parallel_provenance_context(tmp_path: Path) -> None: - raw_id = _seed_invalid_head(tmp_path) - with sqlite3.connect(tmp_path / "source.db") as source, sqlite3.connect(tmp_path / "index.db") as index: - source.execute( - """ - INSERT INTO raw_sessions ( - raw_id, origin, source_path, source_index, blob_hash, blob_size, acquired_at_ms, - logical_source_key, revision_kind, source_revision, - acquisition_generation, revision_authority - ) SELECT ?, origin, source_path, source_index, blob_hash, blob_size, acquired_at_ms, - logical_source_key, 'full', ?, 0, 'quarantined' - FROM raw_sessions WHERE raw_id = ? - """, - ("1" * 64, "2" * 64, raw_id), - ) - index.execute( - """ - INSERT INTO raw_revision_heads ( - logical_source_key, session_id, accepted_raw_id, accepted_source_revision, - accepted_content_hash, accepted_frontier_kind, accepted_frontier, - acquisition_generation, append_end_offset, decided_at_ms - ) SELECT 'chatgpt:parallel-provenance', session_id, ?, ?, - accepted_content_hash, 'semantic', 1, 0, NULL, decided_at_ms - 1 - FROM raw_revision_heads LIMIT 1 - """, - ("3" * 64, "4" * 64), - ) - source.commit() - index.commit() - - report = repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]) - - assert report.eligible_count == 1, report.items[0].reason - assert report.items[0].parallel_session_head_count == 1 - assert report.items[0].quarantined_sibling_raw_count == 1 - assert report.items[0].authority_context_digest - - -def test_quarantined_accepted_raw_repair_source_v7_requires_injective_origin(tmp_path: Path) -> None: - injective_raw = _seed_invalid_head(tmp_path, "injective") - noninjective_raw = _seed_invalid_head(tmp_path, "noninjective") - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute("ALTER TABLE raw_sessions DROP COLUMN capture_mode") - source.execute("PRAGMA user_version = 7") - source.execute("UPDATE raw_sessions SET origin = 'aistudio-drive' WHERE raw_id = ?", (noninjective_raw,)) - source.commit() - - eligible = repair_quarantined_accepted_raws(_config(tmp_path), [injective_raw]) - refused = repair_quarantined_accepted_raws(_config(tmp_path), [noninjective_raw]) - - assert eligible.eligible_count == 1, eligible.items[0].reason - assert eligible.items[0].capture_mode is None - assert refused.ineligible_count == 1 - assert refused.items[0].reason == "source-v7 origin is not injective without capture-mode authority" - - -def test_quarantined_accepted_raw_repair_stages_source_v7_census_before_target_cas(tmp_path: Path) -> None: - raw_id = _seed_invalid_head(tmp_path, "staged") - sibling_raw_id = "e" * 64 - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute( - """ - INSERT INTO raw_sessions ( - raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, - acquired_at_ms, file_mtime_ms, logical_source_key, revision_kind, source_revision, - acquisition_generation, revision_authority - ) SELECT ?, origin, native_id, source_path, source_index, blob_hash, blob_size, - acquired_at_ms, file_mtime_ms, logical_source_key, revision_kind, source_revision, - acquisition_generation, revision_authority - FROM raw_sessions WHERE raw_id = ? - """, - (sibling_raw_id, raw_id), - ) - source.execute( - """ - INSERT INTO blob_refs (blob_hash, ref_id, ref_type, source_path, size_bytes, acquired_at_ms) - SELECT blob_hash, ?, ref_type, source_path, size_bytes, acquired_at_ms - FROM blob_refs WHERE ref_id = ? - """, - (sibling_raw_id, raw_id), - ) - source.execute("DELETE FROM raw_session_memberships WHERE raw_id = ?", (raw_id,)) - source.execute("DELETE FROM raw_membership_census WHERE raw_id = ?", (raw_id,)) - source.execute("ALTER TABLE raw_sessions DROP COLUMN capture_mode") - source.execute("PRAGMA user_version = 7") - source.commit() - - dry_run = repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]) - - assert dry_run.eligible_count == 1, dry_run.items[0].reason - assert dry_run.items[0].census_stage_raw_ids == tuple(sorted((raw_id, sibling_raw_id))) - receipt = tmp_path / "source-v7-stage.jsonl" - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute( - """ - INSERT INTO raw_session_memberships ( - raw_id, logical_source_key, provider_session_id, source_revision, - normalized_content_hash, message_count, acquisition_generation, - revision_authority - ) VALUES (?, 'chatgpt:staged', 'staged', ?, ?, 1, 0, 'quarantined') - """, - ( - sibling_raw_id, - dry_run.items[0].accepted_content_hash, - bytes.fromhex(dry_run.items[0].accepted_content_hash or ""), - ), - ) - source.execute( - """ - INSERT INTO raw_membership_census ( - raw_id, parser_fingerprint, status, member_count, censused_at_ms, detail - ) VALUES (?, 'repair-quarantined-accepted-raw-v1', 'complete', 1, 0, - 'census-only evidence staged before accepted-head authority refinement') - """, - (sibling_raw_id,), - ) - source.commit() - applied = repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - - assert applied.repaired_count == 1 - with sqlite3.connect(tmp_path / "source.db") as source: - assert source.execute( - "SELECT revision_authority, baseline_raw_id FROM raw_sessions WHERE raw_id = ?", (raw_id,) - ).fetchone() == ("byte_proven", raw_id) - assert source.execute( - "SELECT revision_authority, baseline_raw_id FROM raw_sessions WHERE raw_id = ?", (sibling_raw_id,) - ).fetchone() == ("quarantined", None) - assert source.execute( - "SELECT COUNT(*) FROM raw_session_memberships WHERE raw_id IN (?, ?)", (raw_id, sibling_raw_id) - ).fetchone() == (2,) - assert source.execute( - "SELECT COUNT(*) FROM raw_membership_census WHERE raw_id IN (?, ?)", (raw_id, sibling_raw_id) - ).fetchone() == (2,) - reapplied = repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - assert reapplied.repaired_count == 0 - assert reapplied.already_repaired_count == 1 - - -def test_quarantined_accepted_raw_repair_refuses_source_v7_census_staging_with_mixed_cohort(tmp_path: Path) -> None: - raw_id = _seed_invalid_head(tmp_path, "mixed") - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute("DELETE FROM raw_session_memberships WHERE raw_id = ?", (raw_id,)) - source.execute("DELETE FROM raw_membership_census WHERE raw_id = ?", (raw_id,)) - source.execute("UPDATE raw_sessions SET origin = 'aistudio-drive' WHERE raw_id = ?", (raw_id,)) - source.execute("ALTER TABLE raw_sessions DROP COLUMN capture_mode") - source.commit() - - report = repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]) - - assert report.ineligible_count == 1 - assert report.items[0].reason == "source-v7 origin is not injective without capture-mode authority" - - -def test_quarantined_accepted_raw_repair_bounds_source_v7_census_staging_cohort(tmp_path: Path) -> None: - raw_id = _seed_invalid_head(tmp_path, "oversized-cohort") - sibling_raw_id = "d" * 64 - oversized = 256 * 1024 * 1024 + 1 - with sqlite3.connect(tmp_path / "source.db") as source: - source.execute( - """ - INSERT INTO raw_sessions ( - raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, - acquired_at_ms, file_mtime_ms, logical_source_key, revision_kind, source_revision, - acquisition_generation, revision_authority - ) SELECT ?, origin, native_id, source_path, source_index, blob_hash, ?, - acquired_at_ms, file_mtime_ms, logical_source_key, revision_kind, source_revision, - acquisition_generation, revision_authority - FROM raw_sessions WHERE raw_id = ? - """, - (sibling_raw_id, oversized, raw_id), - ) - source.execute( - """ - INSERT INTO blob_refs (blob_hash, ref_id, ref_type, source_path, size_bytes, acquired_at_ms) - SELECT blob_hash, ?, ref_type, source_path, ?, acquired_at_ms - FROM blob_refs WHERE ref_id = ? - """, - (sibling_raw_id, oversized, raw_id), - ) - source.execute("DELETE FROM raw_session_memberships WHERE raw_id = ?", (raw_id,)) - source.execute("DELETE FROM raw_membership_census WHERE raw_id = ?", (raw_id,)) - source.execute("ALTER TABLE raw_sessions DROP COLUMN capture_mode") - source.commit() - - report = repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]) - - assert report.ineligible_count == 1 - assert report.items[0].reason == "same-source-path cohort exceeds the per-raw retained-blob repair limit" - with sqlite3.connect(tmp_path / "source.db") as source: - assert source.execute( - "SELECT COUNT(*) FROM raw_session_memberships WHERE raw_id = ?", (raw_id,) - ).fetchone() == (0,) - assert source.execute("SELECT COUNT(*) FROM raw_membership_census WHERE raw_id = ?", (raw_id,)).fetchone() == ( - 0, - ) - - -def test_quarantined_accepted_raw_repair_bounds_source_v7_census_staging_aggregate(tmp_path: Path) -> None: - raw_id = _seed_invalid_head(tmp_path, "aggregate-cohort") - sibling_size = 171 * 1024 * 1024 - with sqlite3.connect(tmp_path / "source.db") as source: - for sibling_raw_id in ("a" * 64, "b" * 64, "c" * 64): - source.execute( - """ - INSERT INTO raw_sessions ( - raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, - acquired_at_ms, file_mtime_ms, logical_source_key, revision_kind, source_revision, - acquisition_generation, revision_authority - ) SELECT ?, origin, native_id, source_path, source_index, blob_hash, ?, - acquired_at_ms, file_mtime_ms, logical_source_key, revision_kind, source_revision, - acquisition_generation, revision_authority - FROM raw_sessions WHERE raw_id = ? - """, - (sibling_raw_id, sibling_size, raw_id), - ) - source.execute( - """ - INSERT INTO blob_refs (blob_hash, ref_id, ref_type, source_path, size_bytes, acquired_at_ms) - SELECT blob_hash, ?, ref_type, source_path, ?, acquired_at_ms - FROM blob_refs WHERE ref_id = ? - """, - (sibling_raw_id, sibling_size, raw_id), - ) - source.execute("DELETE FROM raw_session_memberships WHERE raw_id = ?", (raw_id,)) - source.execute("DELETE FROM raw_membership_census WHERE raw_id = ?", (raw_id,)) - source.execute("ALTER TABLE raw_sessions DROP COLUMN capture_mode") - source.commit() - - report = repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]) - - assert report.ineligible_count == 1 - assert report.items[0].reason == "same-source-path cohort exceeds the aggregate retained-blob repair limit" - - -def test_quarantined_accepted_raw_repair_rejects_duplicates_and_rolls_back_batch( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - first = _seed_invalid_head(tmp_path, "first") - second = _seed_invalid_head(tmp_path, "second") - with pytest.raises(ValueError, match="duplicate"): - repair_quarantined_accepted_raws(_config(tmp_path), [first, first]) - dry_run = repair_quarantined_accepted_raws(_config(tmp_path), [first, second]) - before = _logical_state(tmp_path, first) - receipt = tmp_path / "rollback-receipt.jsonl" - - from polylogue.storage import repair as repair_module - - original = repair_module._inspect_quarantined_accepted_raw - calls = 0 - - def fail_postproof(archive_root: Path, raw_id: str, *, conn: sqlite3.Connection) -> Any: - nonlocal calls - calls += 1 - item = original(archive_root, raw_id, conn=conn) - if calls == 7: - return repair_module._quarantined_raw_item(item.raw_id, "injected post-proof failure") - return item - - monkeypatch.setattr(repair_module, "_inspect_quarantined_accepted_raw", fail_postproof) - with pytest.raises(RuntimeError, match="terminal state"): - repair_quarantined_accepted_raws( - _config(tmp_path), - [first, second], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - - assert _logical_state(tmp_path, first) == before - assert [json.loads(line)["state"] for line in receipt.read_text().splitlines()] == ["planned"] - - -def test_quarantined_accepted_raw_repair_resumes_planned_receipt_after_committed_source( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - raw_id = _seed_invalid_head(tmp_path) - dry_run = repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]) - receipt = tmp_path / "planned-resume.jsonl" - from polylogue.storage import repair as repair_module - - original_finish = repair_module._finish_quarantined_raw_repair_receipt - monkeypatch.setattr( - repair_module, - "_finish_quarantined_raw_repair_receipt", - lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("injected terminal append crash")), - ) - with pytest.raises(RuntimeError, match="terminal append crash"): - repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - assert [json.loads(line)["state"] for line in receipt.read_text().splitlines()] == ["planned"] - assert _raw_session_row(tmp_path, raw_id)["revision_authority"] == "byte_proven" - - monkeypatch.setattr(repair_module, "_finish_quarantined_raw_repair_receipt", original_finish) - resumed = repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - assert resumed.already_repaired_count == 1 - records = [json.loads(line) for line in receipt.read_text().splitlines()] - assert [record["state"] for record in records] == ["planned", "applied"] - assert records[1]["repaired_raw_ids"] == [] - - -def test_quarantined_accepted_raw_repair_does_not_claim_a_competing_receipt_commit(tmp_path: Path) -> None: - raw_id = _seed_invalid_head(tmp_path) - dry_run = repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]) - interrupted_receipt = tmp_path / "interrupted.jsonl" - competing_receipt = tmp_path / "competing.jsonl" - from polylogue.storage import repair as repair_module - - planned = repair_module._lock_quarantined_raw_repair_receipt(interrupted_receipt, list(dry_run.items)) - planned.close() - competing = repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=competing_receipt, - proof_digest=dry_run.proof_digest, - ) - resumed = repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=interrupted_receipt, - proof_digest=dry_run.proof_digest, - ) - - assert competing.repaired_count == 1 - assert resumed.repaired_count == 0 - assert json.loads(competing_receipt.read_text().splitlines()[-1])["repaired_raw_ids"] == [raw_id] - assert json.loads(interrupted_receipt.read_text().splitlines()[-1])["repaired_raw_ids"] == [] - - -def test_quarantined_accepted_raw_repair_attributes_from_locked_state( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - raw_id = _seed_invalid_head(tmp_path) - dry_run = repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]) - receipt = tmp_path / "pre-lease-competitor.jsonl" - from polylogue.storage import index_generation - - class CompetingRepairLease: - def __init__(self, archive_root: Path) -> None: - self.archive_root = archive_root - - def __enter__(self) -> CompetingRepairLease: - with sqlite3.connect(self.archive_root / "source.db") as source: - source.execute( - """ - UPDATE raw_sessions - SET logical_source_key = ?, revision_kind = 'full', source_revision = ?, - baseline_raw_id = raw_id, acquisition_generation = 0, - revision_authority = 'byte_proven' - WHERE raw_id = ? - """, - ("chatgpt:repair-one", dry_run.items[0].accepted_source_revision, raw_id), - ) - source.commit() - return self - - def __exit__(self, *args: object) -> None: - del args - - monkeypatch.setattr(index_generation, "RebuildLease", CompetingRepairLease) - result = repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - - assert result.repaired_count == 0 - assert result.already_repaired_count == 1 - assert json.loads(receipt.read_text().splitlines()[-1])["repaired_raw_ids"] == [] - - -def test_quarantined_accepted_raw_repair_acquires_exclusive_archive_lock_before_receipt(tmp_path: Path) -> None: - raw_id = _seed_invalid_head(tmp_path) - dry_run = repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]) - receipt = tmp_path / "blocked.jsonl" - - with RebuildLease(tmp_path), pytest.raises(RuntimeError, match="rebuild lease is already held"): - repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - assert not receipt.exists() - - -def test_quarantined_accepted_raw_repair_writes_every_receipt_record_in_full( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - raw_id = _seed_invalid_head(tmp_path) - dry_run = repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]) - receipt = tmp_path / "short-write.jsonl" - from polylogue.storage import repair as repair_module - - original_write = repair_module._receipt_write - - def short_write(descriptor: int, payload: bytes) -> int: - return original_write(descriptor, payload[: max(1, min(11, len(payload)))]) - - monkeypatch.setattr(repair_module, "_receipt_write", short_write) - repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - records = [json.loads(line) for line in receipt.read_text().splitlines()] - assert [record["state"] for record in records] == ["planned", "applied"] - - -def test_quarantined_accepted_raw_repair_recovers_preserved_torn_terminal( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - raw_id = _seed_invalid_head(tmp_path) - dry_run = repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]) - receipt = tmp_path / "torn-terminal.jsonl" - from polylogue.storage import repair as repair_module - - original_finish = repair_module._finish_quarantined_raw_repair_receipt - - def tear_terminal(locked: Any, *, items: list[Any]) -> None: - del items - # A complete JSON prefix without its newline must remain distinguishable - # from the eventual applied record after recovery seals the torn line. - repair_module._write_receipt_all(locked.descriptor, b"{}") - os.fsync(locked.descriptor) - raise RuntimeError("injected torn terminal") - - monkeypatch.setattr(repair_module, "_finish_quarantined_raw_repair_receipt", tear_terminal) - with pytest.raises(RuntimeError, match="torn terminal"): - repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - assert _raw_session_row(tmp_path, raw_id)["revision_authority"] == "byte_proven" - assert not receipt.read_bytes().endswith(b"\n") - - def tear_recovery(locked: Any, *, items: list[Any]) -> None: - del items - if locked.torn_terminals and not locked.receipt_terminated: - repair_module._write_receipt_all(locked.descriptor, b"\xff\n") - repair_module._write_receipt_all(locked.descriptor, b'{"state":') - os.fsync(locked.descriptor) - raise RuntimeError("injected torn recovery terminal") - - monkeypatch.setattr(repair_module, "_finish_quarantined_raw_repair_receipt", tear_recovery) - with pytest.raises(RuntimeError, match="torn recovery terminal"): - repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - - monkeypatch.setattr(repair_module, "_finish_quarantined_raw_repair_receipt", original_finish) - repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - lines = receipt.read_bytes().splitlines() - assert len(lines) == 4 - assert lines[1] == b"{}\xff" - assert lines[2] == b'{"state":\xff' - recovered = json.loads(lines[3]) - assert recovered["state"] == "applied" - assert recovered["torn_terminals"] == [ - {"bytes": len(fragment), "sha256": hashlib.sha256(fragment).hexdigest()} for fragment in lines[1:3] - ] - - reapplied = repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - assert reapplied.already_repaired_count == 1 - - -def test_quarantined_accepted_raw_repair_rejects_torn_terminal_without_source_commit(tmp_path: Path) -> None: - raw_id = _seed_invalid_head(tmp_path) - dry_run = repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]) - receipt_path = tmp_path / "false-torn-terminal.jsonl" - from polylogue.storage import repair as repair_module - - locked = repair_module._lock_quarantined_raw_repair_receipt(receipt_path, list(dry_run.items)) - repair_module._write_receipt_all(locked.descriptor, b'{"state":') - os.fsync(locked.descriptor) - locked.close() - - with pytest.raises(RuntimeError, match="no matching committed source refinement"): - repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt_path, - proof_digest=dry_run.proof_digest, - ) - assert _raw_session_row(tmp_path, raw_id)["revision_authority"] == "quarantined" - - -@pytest.mark.parametrize( - ("record_index", "field", "value"), - [ - (0, "state", "applied"), - (0, "planned_at_ms", "not-an-int"), - (1, "schema", "wrong"), - (1, "target_hash", "0" * 64), - (1, "proven_raw_ids", []), - (1, "repaired_raw_ids", ["0" * 64]), - ], -) -def test_quarantined_accepted_raw_repair_rejects_corrupt_receipt_records( - tmp_path: Path, - record_index: int, - field: str, - value: object, -) -> None: - raw_id = _seed_invalid_head(tmp_path) - dry_run = repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]) - receipt = tmp_path / "corrupt.jsonl" - repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - records = [json.loads(line) for line in receipt.read_text().splitlines()] - records[record_index][field] = value - receipt.write_text("".join(json.dumps(record) + "\n" for record in records)) - - with pytest.raises(RuntimeError, match="receipt"): - repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt, - proof_digest=dry_run.proof_digest, - ) - - -def test_quarantined_accepted_raw_repair_serializes_one_receipt_inode(tmp_path: Path) -> None: - raw_id = _seed_invalid_head(tmp_path) - dry_run = repair_quarantined_accepted_raws(_config(tmp_path), [raw_id]) - receipt_path = tmp_path / "locked.jsonl" - from polylogue.storage import repair as repair_module - - locked = repair_module._lock_quarantined_raw_repair_receipt(receipt_path, list(dry_run.items)) - try: - with pytest.raises(RuntimeError, match="already locked"): - repair_quarantined_accepted_raws( - _config(tmp_path), - [raw_id], - apply=True, - receipt_path=receipt_path, - proof_digest=dry_run.proof_digest, - ) - assert _raw_session_row(tmp_path, raw_id)["revision_authority"] == "quarantined" - finally: - locked.close() - - -def test_quarantined_accepted_raw_repair_cas_failure_rolls_back_entire_batch( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - first = _seed_invalid_head(tmp_path, "cas-first") - second = _seed_invalid_head(tmp_path, "cas-second") - dry_run = repair_quarantined_accepted_raws(_config(tmp_path), [first, second]) - before = {raw_id: _raw_session_row(tmp_path, raw_id) for raw_id in (first, second)} - from polylogue.storage import repair as repair_module - - original = repair_module._cas_refine_quarantined_accepted_raw - calls = 0 - - def fail_second(conn: sqlite3.Connection, item: Any) -> None: - nonlocal calls - calls += 1 - if calls == 2: - raise RuntimeError("injected CAS failure") - original(conn, item) - - monkeypatch.setattr(repair_module, "_cas_refine_quarantined_accepted_raw", fail_second) - with pytest.raises(RuntimeError, match="injected CAS failure"): - repair_quarantined_accepted_raws( - _config(tmp_path), - [first, second], - apply=True, - receipt_path=tmp_path / "cas-rollback.jsonl", - proof_digest=dry_run.proof_digest, - ) - assert {raw_id: _raw_session_row(tmp_path, raw_id) for raw_id in (first, second)} == before - - -@pytest.mark.parametrize("limit_kind", ["target", "aggregate"]) -def test_quarantined_accepted_raw_repair_checks_blob_budget_before_read( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - limit_kind: str, -) -> None: - raw_ids = [_seed_invalid_head(tmp_path, "budget-first")] - if limit_kind == "aggregate": - raw_ids.append(_seed_invalid_head(tmp_path, "budget-second")) - from polylogue.storage import repair as repair_module - - blob_sizes = [int(str(_raw_session_row(tmp_path, raw_id)["blob_size"])) for raw_id in raw_ids] - if limit_kind == "target": - monkeypatch.setattr(repair_module, "_QUARANTINED_ACCEPTED_RAW_REPAIR_BLOB_LIMIT_BYTES", int(blob_sizes[0]) - 1) - else: - monkeypatch.setattr( - repair_module, "_QUARANTINED_ACCEPTED_RAW_REPAIR_TOTAL_BLOB_LIMIT_BYTES", sum(blob_sizes) - 1 - ) - monkeypatch.setattr(BlobStore, "read_all", lambda *args, **kwargs: pytest.fail("blob was read before budget check")) - - with pytest.raises(RuntimeError, match="blob limit"): - repair_quarantined_accepted_raws(_config(tmp_path), raw_ids)