diff --git a/docs/features/hosted-worker.md b/docs/features/hosted-worker.md index 2cf9ec4..5eaf12e 100644 --- a/docs/features/hosted-worker.md +++ b/docs/features/hosted-worker.md @@ -43,6 +43,11 @@ TypeSafe API's shape, for GitHub Actions OIDC tokens (`worker/src/oidc.ts`) and `defrag_log`. `markVerified` mirrors `MemoryStore.mark_verified`: change one, change both. - The operator `snapshot` op is `kypp sync`'s read-only pull: recall's visibility with every candidate (`visible(..., "all")`), paged by id. A field the offline cache needs must be in `CLAIM_COLUMNS`. +- Operator `remember`/`correct` take an optional idempotency `key`. Its `operator_receipts` row goes + in the claim's own batch, so a repeat (or a racing send whose batch lost on the primary key) returns + the first claim with `replayed: true` and its current `status`; the same key with a different write is refused. Receipts are + kept for good: a write queued offline may be resent at any later sync. The `capabilities` op + (`idempotent_writes: true`) is how the CLI tells this Worker from one that ignores `key`. - Hosted operator `correct` is an explicit replacement in the existing `memory_claims` store, not a rank contest. Insert, FTS/pointers, exact-group supersession and audit share one D1 batch; the returned superseded count comes from that transaction. Equal clocks, query order and lower diff --git a/docs/features/offline.md b/docs/features/offline.md index 214cfa8..2dc7dc3 100644 --- a/docs/features/offline.md +++ b/docs/features/offline.md @@ -20,12 +20,19 @@ operator's view of the project (`POST /v1/operator` `snapshot`, 500 claims a pag - Only `remote.Unreachable` (no connection was made) falls back to the cache or queue. urllib wraps a failure while sending in `URLError` too, so `remote.py` records whether `connect()` finished (`_conn.made`); after it, a failure is ambiguous and never falls back. -- A queued write is marked sent before its request and deleted only after a 200. Only `Unreachable` - unmarks it. Anything else (an error answer, a timeout, a dropped connection) may follow a committed - write: the Worker answers 400 for any failure, even after a commit, and operator `remember` has no - idempotency key. A marked write blocks the push until the operator checks and runs `kypp sync - --drop N` or `--resend N`; it is never re-sent on its own (a purge must not be undone by a replay). - The pull still runs, so a stuck write doesn't freeze the copy. +- Before a push, `kypp sync` asks the Worker for `capabilities`. Against one that honours keys, each + queued write gets an idempotency key, saved on disk before its first send and sent again on every + retry. The Worker stores the key in `operator_receipts` in the claim's own batch and answers a repeat + with the first claim, so a write is resent automatically after any failure (an error answer, a + timeout, a lost reply) and still lands once; a purged claim is never written back by a late resend. + A write leaves the queue only after a 200 and blocks the writes behind it until then; a write the + Worker keeps refusing stays until `kypp sync --drop N`. The pull still runs, so a stuck write doesn't + freeze the copy. +- A Worker deployed before keys answers `unknown op` to `capabilities` and ignores `key`, so the write + goes without one, and a row's key is cleared before any such send. A row sent without a key (to such a Worker, or by an older CLI), or a keyed row the + Worker no longer honours, may or may not have landed: it blocks the push until the operator checks and + runs `--drop N` or `--resend N`. A send that never connected restores the row's earlier mark (none if + it was never sent), so an earlier, maybe committed send stays on record. - The cache is a copy, never an authority: nothing in `offline.py` accepts, promotes or supersedes. The Worker applies its own rules when a queued write arrives. - `correct`, `reject` and `purge` are refused offline (`shell._online`): they replace or retire memory diff --git a/kypp/offline.py b/kypp/offline.py index 709a972..3f63ab5 100644 --- a/kypp/offline.py +++ b/kypp/offline.py @@ -13,6 +13,7 @@ import json import os import re +import secrets import sqlite3 import sys import time @@ -33,7 +34,7 @@ CREATE TABLE IF NOT EXISTS forgotten (url TEXT NOT NULL, id TEXT NOT NULL, purged INTEGER NOT NULL, at REAL NOT NULL, PRIMARY KEY (url, id)); CREATE TABLE IF NOT EXISTS outbox (seq INTEGER PRIMARY KEY AUTOINCREMENT, url TEXT NOT NULL, project TEXT NOT NULL, - op TEXT NOT NULL, args TEXT NOT NULL, queued_at TEXT NOT NULL, sent_at TEXT); + op TEXT NOT NULL, args TEXT NOT NULL, queued_at TEXT NOT NULL, sent_at TEXT, key TEXT); """ @@ -56,6 +57,16 @@ def _db() -> sqlite3.Connection: os.umask(old) os.chmod(path, 0o600) db.executescript(_SCHEMA) + # A cache made before writes carried keys gains the column; push gives each row its key. The check + # and the ALTER share one write transaction, so two CLIs opening an old cache can't both add it. + db.execute("BEGIN IMMEDIATE") + try: + if "key" not in {r[1] for r in db.execute("PRAGMA table_info(outbox)")}: + db.execute("ALTER TABLE outbox ADD COLUMN key TEXT") + db.execute("COMMIT") + except BaseException: + db.execute("ROLLBACK") + raise return db @@ -192,32 +203,56 @@ def forget(claim_id: str, purged: bool) -> None: db.execute("DELETE FROM claims WHERE url = ? AND id = ?", (_url(), claim_id)) +def _keyed() -> bool: + """Whether the Worker honours idempotency keys. One deployed before them ignores `key`.""" + try: + return remote.call("capabilities").get("idempotent_writes") is True + except remote.Unreachable: + raise + except SystemExit as e: + if "unknown op" in str(e.code): + return False + raise + + def push(db: sqlite3.Connection) -> int: - """Send queued writes in order. Each is marked sent before its request and leaves the queue only - after a 200. Only a request that never got a connection is unmarked again. Any other failure (an - error answer, a timeout, a dropped connection) may follow a committed write: the Worker answers 400 - for any failure, even after a commit, and has no idempotency key. So a marked write is never sent - again on its own; the operator checks the hosted store and runs `--drop N` or `--resend N`.""" - sent = 0 - for seq, op, raw, sent_at in db.execute( - "SELECT seq, op, args, sent_at FROM outbox WHERE url = ? ORDER BY seq", (_url(),)).fetchall(): + """Send queued writes in order. Against a Worker that honours keys, each is sent under its own + idempotency key, so it is applied at most once however often it is sent: a write leaves the queue + only after a 200, and any other answer (or none) leaves it, and the writes after it, for the next + `kypp sync` to send again. A row sent without a key (to an older Worker, or by an older CLI) may or + may not have landed, so it is never resent on its own: the operator decides it.""" + sent, keyed = 0, None + for seq, op, raw, sent_at, key in db.execute( + "SELECT seq, op, args, sent_at, key FROM outbox WHERE url = ? ORDER BY seq", (_url(),)).fetchall(): if op not in _QUEUEABLE: raise SystemExit(f"kypp: queued write #{seq} is `{op}`, which is never queued; " f"drop it with `kypp sync --drop {seq}`") stored = json.loads(raw) args = {k: stored.get(k) for k in _QUEUEABLE[op]} - if sent_at: - raise SystemExit(_unsure(seq, args, f"it was sent at {sent_at} and no 200 came back")) + # A row keeps a key only while every send of it went to a Worker that honours keys: it is fixed on + # disk before such a send and cleared before any other, so only a keyed row is resent on its own. + if keyed is None: + keyed = _keyed() + if sent_at and not (key and keyed): + why = "and the Worker no longer honours idempotency keys" if key else "without an idempotency key" + raise SystemExit(_unsure(seq, args, f"it was sent at {sent_at} {why} and no 200 came back")) + key = (key or secrets.token_hex(16)) if keyed else None with db: - db.execute("UPDATE outbox SET sent_at = ? WHERE seq = ?", (_now(), seq)) + db.execute("UPDATE outbox SET sent_at = ?, key = ? WHERE seq = ?", (_now(), key, seq)) try: - remote.call(op, **args) + remote.call(op, key=key, **args) except remote.Unreachable: - with db: # no connection was made, so nothing was applied - db.execute("UPDATE outbox SET sent_at = NULL WHERE seq = ?", (seq,)) + with db: # this attempt never connected; an earlier send may still have landed, so keep its mark + db.execute("UPDATE outbox SET sent_at = ? WHERE seq = ?", (sent_at, seq)) raise except SystemExit as e: - raise SystemExit(f"{e.code}\n" + _unsure(seq, args, "the Worker didn't answer 200")) from None + if not key: + raise SystemExit(f"{e.code}\n" + _unsure(seq, args, "the Worker (which predates idempotency keys) " + "didn't answer 200")) from None + raise SystemExit(f"{e.code}\nkypp: queued write #{seq} ({args.get('subject')!r}) and the writes after " + f"it stay queued; the next `kypp sync` sends it again under the same key, so it lands " + f"at most once. If the Worker refused it, fix the cause or run " + f"`kypp sync --drop {seq}`.") from None with db: db.execute("DELETE FROM outbox WHERE seq = ?", (seq,)) sent += 1 @@ -252,7 +287,8 @@ def sync_main(): ap.add_argument("--drop", type=int, action="append", default=[], metavar="N", help="discard queued write #N without sending it (it reached the store, or it was refused)") ap.add_argument("--resend", type=int, action="append", default=[], metavar="N", - help="send queued write #N again: you checked that the hosted store doesn't have it") + help="send queued write #N again (one sent without an idempotency key): you checked that the " + "hosted store doesn't have it") args = ap.parse_args() if not remote.remote_url(): raise SystemExit("kypp sync: set KYPP_URL; the local store needs no sync") diff --git a/tests/test_offline.py b/tests/test_offline.py index bdeb7d0..0376f5d 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -9,6 +9,7 @@ import sys import tempfile import threading +import time import unittest import urllib.error from unittest import mock @@ -28,6 +29,9 @@ class _Worker(http.server.BaseHTTPRequestHandler): refuse: set = set() # ops answered 400 page = 2 claim_reply: dict = {} + applied: dict = {} # idempotency key -> claim id, as the Worker's operator_receipts + old = False # a Worker deployed before idempotency keys: no `capabilities`, `key` ignored + def do_POST(self): body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) @@ -40,8 +44,17 @@ def do_POST(self): rest = [c for c in self.claims if c["id"] > body.get("after", "")] got = rest[:self.page] return self._send(200, {"claims": got, "next": got[-1]["id"] if len(rest) > self.page else None}) + if body["op"] == "capabilities": + if self.old: + return self._send(400, {"error": 'unknown op "capabilities"'}) + return self._send(200, {"idempotent_writes": True}) + if body["op"] == "remember" and self.old: + self.applied[len(self.applied)] = body["subject"] + return self._send(200, {"id": "e" * 32, "status": "candidate"}) if body["op"] == "remember": - return self._send(200, {"id": "f" * 32, "status": "candidate"}) + replayed = body.get("key") in self.applied + cid = self.applied.setdefault(body.get("key"), f"{len(self.applied):032x}") + return self._send(200, {"id": cid, "status": "candidate", "key": body.get("key"), "replayed": replayed}) if body["op"] in ("purge", "reject"): return self._send(200, self.claim_reply) return self._send(200, {"text": "online answer"}) @@ -77,7 +90,7 @@ def _run(main, *argv) -> tuple[str, str]: class OfflineTest(unittest.TestCase): def setUp(self): _Worker.claims = [_claim(1), _claim(2, status="candidate"), _claim(3, type="pitfall")] - _Worker.seen, _Worker.refuse, _Worker.page = [], set(), 2 + _Worker.seen, _Worker.refuse, _Worker.page, _Worker.applied, _Worker.old = [], set(), 2, {}, False self.srv = http.server.HTTPServer(("127.0.0.1", 0), _Worker) threading.Thread(target=self.srv.serve_forever, daemon=True).start() self.addCleanup(self.srv.server_close) @@ -178,57 +191,246 @@ def test_queued_writes_leave_the_queue_only_once_the_worker_answers(self): out, _ = self.sync() self.assertIn("sent 0 queued write(s)", out) - def test_a_write_without_a_200_is_never_sent_again_on_its_own(self): - # An error answer, a gateway error or a lost reply may follow a committed write (the Worker answers - # 400 even after a commit, and has no idempotency key), so only the operator decides. + def test_a_write_without_a_200_is_sent_again_under_its_key_and_lands_once(self): + # A refusal, a gateway error or a lost reply after a commit: the next sync resends the same key, + # and the Worker returns the first claim instead of inserting another. for failure in ("refused", "lost"): with self.subTest(failure=failure): self.setUp() dead = _dead_url() self.offline_copy(dead) with self.env_at(dead): - _run(shell.remember_main, "leaked", "the token is abc") + _run(shell.remember_main, "a", "first") _run(shell.remember_main, "b", "second") self.back_online() - orig = remote.call + orig, fails = remote.call, [1] - def fail(op, **args): - if op == "remember": - _Worker.seen.append({"op": op, **args}) + def flaky(op, **args): + if op == "remember" and fails: + fails.pop() + if failure == "lost": + orig(op, **args) # committed; the answer never arrives raise SystemExit("kypp: answered 400" if failure == "refused" else "kypp: timed out") return orig(op, **args) - with self.env_at(self.url), mock.patch.object(remote, "call", fail): + with self.env_at(self.url), mock.patch.object(remote, "call", flaky): with self.assertRaises(SystemExit) as e: _run(offline.sync_main) - self.assertIn("may already be in the hosted store", str(e.exception.code)) + self.assertIn("sends it again under the same key", str(e.exception.code)) self.assertIn("cached", str(e.exception.code)) # the copy still refreshed - for _ in range(2): # e.g. the operator purged it meanwhile: still not re-sent - with self.assertRaises(SystemExit) as e: - self.sync() - self.assertIn("--resend 1", str(e.exception.code)) - self.assertEqual(len([b for b in _Worker.seen if b["op"] == "remember"]), 1) with self.env_at(self.url): - out, _ = _run(offline.sync_main, "--drop", "1") - self.assertIn("dropped queued write #1", out) - self.assertIn("sent 1 queued write(s)", out) # "b" goes once the operator decided - - def test_resend_sends_a_write_the_operator_checked(self): + self.assertEqual(len(offline._queued(offline._db(), "p")), 2) # "b" waits behind "a" + out, _ = self.sync() + self.assertIn("sent 2 queued write(s)", out) + sent = [b for b in _Worker.seen if b["op"] == "remember"] + self.assertEqual([b["subject"] for b in sent], ["a", "a", "b"] if failure == "lost" else ["a", "b"]) + self.assertEqual(len({b["key"] for b in sent}), 2) # a resend reuses its key + self.assertEqual(len(_Worker.applied), 2) + + def test_a_row_sent_before_keys_waits_for_the_operator(self): dead = _dead_url() self.offline_copy(dead) with self.env_at(dead): _run(shell.remember_main, "a", "first") + _run(shell.remember_main, "b", "second") self.back_online() with self.env_at(self.url): db = offline._db() - db.execute("UPDATE outbox SET sent_at = 'earlier'") + db.execute("UPDATE outbox SET key = NULL") # as a cache from before keys + db.execute("UPDATE outbox SET sent_at = 'earlier' WHERE seq = 1") db.commit() + for _ in range(2): + with self.assertRaises(SystemExit) as e: + self.sync() + self.assertIn("without an idempotency key", str(e.exception.code)) + self.assertIn("--resend 1", str(e.exception.code)) + self.assertEqual([b for b in _Worker.seen if b["op"] == "remember"], []) + with self.env_at(self.url): + out, _ = _run(offline.sync_main, "--resend", "1") + self.assertIn("sent 2 queued write(s)", out) # an unsent row from before keys gets one + self.assertTrue(all(b.get("key") for b in _Worker.seen if b["op"] == "remember")) + + def test_a_lost_reply_from_a_worker_without_keys_is_never_resent_on_its_own(self): + # A Worker deployed before keys ignores `key`; a resend after a committed write would duplicate it. + _Worker.old = True + dead = _dead_url() + self.offline_copy(dead) + with self.env_at(dead): + _run(shell.remember_main, "a", "first") + self.back_online() + orig, fails = remote.call, [1] + + def lost(op, **args): + if op == "remember" and fails: + fails.pop() + orig(op, **args) # committed; the answer never arrives + raise SystemExit("kypp: timed out") + return orig(op, **args) + + with self.env_at(self.url), mock.patch.object(remote, "call", lost): + with self.assertRaises(SystemExit) as e: + _run(offline.sync_main) + self.assertIn("predates idempotency keys", str(e.exception.code)) + _Worker.old = False # even once the Worker is upgraded, the unkeyed send stays the operator's call + for _ in range(2): + with self.assertRaises(SystemExit) as e: + self.sync() + self.assertIn("--resend 1", str(e.exception.code)) + self.assertEqual(len(_Worker.applied), 1) + self.assertNotIn("key", [k for b in _Worker.seen if b["op"] == "remember" for k in b]) + + def test_a_keyed_row_is_not_resent_to_a_worker_that_stopped_honouring_keys(self): + dead = _dead_url() + self.offline_copy(dead) + with self.env_at(dead): + _run(shell.remember_main, "a", "first") + self.back_online() + _Worker.refuse = {"remember"} with self.assertRaises(SystemExit): self.sync() + _Worker.refuse, _Worker.old = set(), True + with self.assertRaises(SystemExit) as e: + self.sync() + self.assertIn("no longer honours idempotency keys", str(e.exception.code)) + self.assertEqual(len([b for b in _Worker.seen if b["op"] == "remember"]), 1) + + def test_a_resend_that_cannot_connect_keeps_the_earlier_send_on_record(self): + dead = _dead_url() + self.offline_copy(dead) + with self.env_at(dead): + _run(shell.remember_main, "a", "first") + self.back_online() + orig, fails = remote.call, [1] + + def lost(op, **args): + if op == "remember" and fails: + fails.pop() + orig(op, **args) # committed; the answer never arrives + raise SystemExit("kypp: timed out") + return orig(op, **args) + + with self.env_at(self.url), mock.patch.object(remote, "call", lost): + with self.assertRaises(SystemExit): + _run(offline.sync_main) + + def offline_resend(op, **args): + if op == "remember": + raise remote.Unreachable("kypp: can't reach it") + return orig(op, **args) + + with self.env_at(self.url), mock.patch.object(remote, "call", offline_resend): + with self.assertRaises(SystemExit): + _run(offline.sync_main) + _Worker.old = True # the Worker is downgraded: the row was sent once, so it stays the operator's call + with self.assertRaises(SystemExit) as e: + self.sync() + self.assertIn("no longer honours idempotency keys", str(e.exception.code)) + self.assertEqual(len(_Worker.applied), 1) + + def test_a_row_sent_to_a_worker_without_keys_loses_its_key(self): + # Keyed Worker, the send can't connect; downgraded Worker, the reply is lost; upgraded again. + dead = _dead_url() + self.offline_copy(dead) + with self.env_at(dead): + _run(shell.remember_main, "a", "first") + self.back_online() + orig = remote.call + + def unreachable(op, **args): + if op == "remember": + raise remote.Unreachable("kypp: can't reach it") + return orig(op, **args) + + with self.env_at(self.url), mock.patch.object(remote, "call", unreachable): + with self.assertRaises(SystemExit): + _run(offline.sync_main) + _Worker.old = True + + def lost(op, **args): + res = orig(op, **args) + if op == "remember": + raise SystemExit("kypp: timed out") + return res + + with self.env_at(self.url), mock.patch.object(remote, "call", lost): + with self.assertRaises(SystemExit): + _run(offline.sync_main) + _Worker.old = False + with self.assertRaises(SystemExit) as e: + self.sync() + self.assertIn("without an idempotency key", str(e.exception.code)) + self.assertEqual(len(_Worker.applied), 1) + + def test_a_cache_from_before_keys_gains_the_column(self): + import sqlite3 + path = self.env["KYPP_CACHE_DB"] + old = sqlite3.connect(path) + old.execute("CREATE TABLE outbox (seq INTEGER PRIMARY KEY AUTOINCREMENT, url TEXT NOT NULL, project TEXT " + "NOT NULL, op TEXT NOT NULL, args TEXT NOT NULL, queued_at TEXT NOT NULL, sent_at TEXT)") + old.execute("INSERT INTO outbox(url, project, op, args, queued_at) VALUES(?, 'p', 'remember', ?, 'x')", + (self.url, json.dumps({"subject": "old", "content": "row", "project": "p"}))) + old.commit() + old.close() + out, _ = self.sync() + self.assertIn("sent 1 queued write(s)", out) + + def test_clis_opening_an_old_cache_at_once_add_the_column_once(self): + import sqlite3 + path = self.env["KYPP_CACHE_DB"] + connect = sqlite3.connect + + class Slow(sqlite3.Connection): + # Widens the gap between reading the schema and changing it, where a second CLI could step in. + def execute(self, sql, *a): + cur = super().execute(sql, *a) + if sql.startswith("PRAGMA table_info"): + time.sleep(0.05) + return cur + + def slow(*a, **kw): + return connect(*a, factory=Slow, **kw) + + for round_ in range(3): + with contextlib.suppress(FileNotFoundError): + os.remove(path) + old = sqlite3.connect(path) + old.execute("CREATE TABLE outbox (seq INTEGER PRIMARY KEY AUTOINCREMENT, url TEXT NOT NULL, project " + "TEXT NOT NULL, op TEXT NOT NULL, args TEXT NOT NULL, queued_at TEXT NOT NULL, sent_at TEXT)") + old.commit() + old.close() + gate, errors = threading.Barrier(6), [] + + def open_it(): + gate.wait() + try: + offline._db().close() + except Exception as e: # noqa: BLE001 - the test reports any failure + errors.append(e) + + with mock.patch.dict(os.environ, self.env), mock.patch.object(offline.sqlite3, "connect", slow): + threads = [threading.Thread(target=open_it) for _ in range(6)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertEqual(errors, [], f"round {round_}") + + def test_drop_discards_a_write_the_worker_keeps_refusing(self): + dead = _dead_url() + self.offline_copy(dead) + with self.env_at(dead): + _run(shell.remember_main, "a", "first") + _run(shell.remember_main, "b", "second") + self.back_online() + _Worker.refuse = {"remember"} + with self.assertRaises(SystemExit) as e: + self.sync() + self.assertIn("--drop 1", str(e.exception.code)) + _Worker.refuse = set() with self.env_at(self.url): - out, _ = _run(offline.sync_main, "--resend", "1") + out, _ = _run(offline.sync_main, "--drop", "1") + self.assertIn("dropped queued write #1", out) self.assertIn("sent 1 queued write(s)", out) - self.assertEqual(len([b for b in _Worker.seen if b["op"] == "remember"]), 1) def test_a_tampered_queue_row_never_becomes_another_operator_write(self): dead = _dead_url() @@ -245,7 +447,7 @@ def test_a_tampered_queue_row_never_becomes_another_operator_write(self): with self.assertRaises(SystemExit) as e: self.sync() self.assertIn("never queued", str(e.exception.code)) - sent = [b for b in _Worker.seen if b["op"] not in ("snapshot",)] + sent = [b for b in _Worker.seen if b["op"] not in ("snapshot", "capabilities")] self.assertEqual([b["op"] for b in sent], ["remember"]) self.assertNotIn("authority", sent[0]) diff --git a/worker/migrations/0012_operator_receipts.sql b/worker/migrations/0012_operator_receipts.sql new file mode 100644 index 0000000..2903c13 --- /dev/null +++ b/worker/migrations/0012_operator_receipts.sql @@ -0,0 +1,5 @@ +-- One row per operator write sent with an idempotency key: a resend of that key returns this claim +-- instead of inserting another. Kept for good, so a write queued offline can be resent at any later sync. +CREATE TABLE IF NOT EXISTS operator_receipts ( + key TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, claim_id TEXT NOT NULL, created_at TEXT NOT NULL +); diff --git a/worker/src/operator.ts b/worker/src/operator.ts index d3bbea7..f926222 100644 --- a/worker/src/operator.ts +++ b/worker/src/operator.ts @@ -15,6 +15,8 @@ const OPERATOR_USER = "owner"; const SNAPSHOT_PAGE = 500; // A repo's bare name, as `resolveOrigin` matches the last segment of an origin like `owner/repo`. const PROJECT_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$/; +// An idempotency key the client makes once per write and sends again on every retry of it. +const WRITE_KEY = /^[A-Za-z0-9_-]{16,128}$/; type Args = Record; export interface OperatorOptions { s1?: S1Client | null; policy?: Policy } @@ -63,13 +65,17 @@ async function run(store: D1Store, a: Args, opts: OperatorOptions): Promise { + // With a key, the write lands at most once: the receipt is inserted in the claim's own batch, so a + // resend of the key (after a lost answer) returns the first claim instead of inserting another. + async operatorClaim(c: ClaimInput, op: "remember" | "correct", verify: string | null, key: string | null = null): Promise<{ id: string; superseded: number; replayed: boolean }> { if (!TYPES.includes(c.type)) throw new Error(`bad type ${c.type}`); if (!SCOPES.includes(c.scope) || c.scope === "group" || c.scope === "room") throw new Error(`bad scope ${c.scope}`); if (c.scope === "project" && !c.project) throw new Error("a project-scoped operator claim needs a known project"); @@ -246,7 +248,20 @@ export class D1Store { const project = c.scope === "project" ? c.project : null; const confidence = Math.min(1, Math.max(0, Number(c.confidence) || 0)); const codeRefs = cleanCodeRefs(c.codeRefs); + const fingerprint = key ? await sha256(JSON.stringify([op, c.type, c.subject, c.content, c.scope, project, + c.project, confidence, !!c.accept, verify, c.agent, c.user])) : ""; + const replay = async () => { + const r = await this.db.prepare("SELECT fingerprint, claim_id FROM operator_receipts WHERE key = ?").bind(key).first<{ fingerprint: string; claim_id: string }>(); + if (r && r.fingerprint !== fingerprint) throw new Error(`key ${JSON.stringify(key)} was already used for a different write`); + return r ? { id: r.claim_id, superseded: 0, replayed: true } : null; + }; + if (key) { + const prior = await replay(); + if (prior) return prior; + } const statements = [ + ...(key ? [this.db.prepare("INSERT INTO operator_receipts(key, fingerprint, claim_id, created_at) VALUES(?,?,?,?)") + .bind(key, fingerprint, id, ts)] : []), this.db.prepare( "INSERT INTO memory_claims(id,type,subject,content,scope,project,origin,session,agent,user,status,authority,confidence,source_ids,code_refs,metadata,verify,created_at,updated_at)" + " VALUES(?,?,?,?,?,?,?,NULL,?,?,?,?,?,'[]',?,'{}',?,?,?)", @@ -271,8 +286,16 @@ export class D1Store { this.db.prepare(`UPDATE memory_claims SET status='superseded', updated_at=? WHERE ${prior}`).bind(ts, ...key), ); } - const results = await this.db.batch(statements); - return { id, superseded: op === "correct" ? results[results.length - 1].meta.changes : 0 }; + let results; + try { + results = await this.db.batch(statements); + } catch (e) { + // A concurrent send of the same key committed first: the receipt's primary key rolled this batch back. + const prior = key ? await replay() : null; + if (prior) return prior; + throw e; + } + return { id, superseded: op === "correct" ? results[results.length - 1].meta.changes : 0, replayed: false }; } // Every claim the operator's `kypp verify` re-checks in this project: any status, so a rejected @@ -737,3 +760,8 @@ export class D1Store { return { applied, stale }; } } + +async function sha256(value: string): Promise { + const bytes = new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value))); + return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join(""); +} diff --git a/worker/test/operator.test.ts b/worker/test/operator.test.ts index 49d01d0..1acb1c3 100644 --- a/worker/test/operator.test.ts +++ b/worker/test/operator.test.ts @@ -287,3 +287,69 @@ test("snapshot pages the operator's live view of one project by id, without reje assert.deepEqual(new Set([...first.body!.claims, ...rest.body!.claims].map((c: any) => c.status)), new Set(["accepted", "candidate"])); assert.equal((await call({ op: "snapshot", project: "nope" })).status, 400); }); + +test("a remember resent with its key lands once, and a key reused for another write is refused", async () => { + const { db, call, row } = await setup(); + const write = { op: "remember", subject: "deploy", content: "ship on green", project: "p", key: "k".repeat(24) }; + const first = await call(write); + assert.equal(first.status, 200); + assert.equal(first.body!.replayed, false); + const again = await call(write); + assert.equal(again.status, 200); + assert.deepEqual(again.body, { id: first.body!.id, status: "candidate", key: write.key, replayed: true }); + const n = async () => (await db.prepare("SELECT count(*) AS n FROM memory_claims WHERE subject = 'deploy'").first() as any).n; + assert.equal(await n(), 1); + const other = await call({ ...write, content: "ship any time" }); + assert.equal(other.status, 400); + assert.match(other.body!.error, /already used for a different write/); + assert.equal(await n(), 1); + // A purged claim is not written back by a late resend of its key. + await call({ op: "purge", handle: first.body!.id, reason: "leaked" }); + const late = await call(write); + assert.equal(late.body!.replayed, true); + assert.equal(late.body!.status, "rejected"); + assert.equal(await n(), 0); + assert.equal((await row(first.body!.id)).content, "[purged]"); +}); + +test("a send of a key that loses the race to another send of it returns the winner's claim", async () => { + const db = memoryD1(); + const input = { type: "fact" as const, subject: "race", content: "one copy", scope: "global" as const, project: null, + confidence: 0.7, sourceIds: [], codeRefs: [], accept: false, agent: "kypp-cli", user: "owner" }; + const key = "r".repeat(24); + let winner = ""; + // The other send commits after this one checked for a receipt and before its batch runs. + const racing = new Proxy(db, { get: (t, p) => p !== "batch" ? (t as any)[p] : async (stmts: unknown[]) => { + if (!winner) winner = (await new D1Store(db).operatorClaim(input, "remember", null, key)).id; + return t.batch(stmts as any); + } }); + const loser = await new D1Store(racing).operatorClaim(input, "remember", null, key); + assert.deepEqual(loser, { id: winner, superseded: 0, replayed: true }); + assert.equal((await db.prepare("SELECT count(*) AS n FROM memory_claims WHERE subject = 'race'").first() as any).n, 1); +}); + +test("a malformed key is refused before anything is written, and no key keeps the old answer", async () => { + const { call } = await setup(); + const bad = await call({ op: "remember", subject: "s", content: "c", project: "p", key: "short" }); + assert.equal(bad.status, 400); + const plain = await call({ op: "remember", subject: "s", content: "c", project: "p" }); + assert.deepEqual(Object.keys(plain.body!).sort(), ["id", "status"]); +}); + +test("a keyed correct resent after a purge reports the claim's current status and is not written back", async () => { + const { call, row } = await setup(); + const write = { op: "correct", subject: "deploy", content: "ship on green", project: "p", key: "c".repeat(24) }; + const first = await call(write); + assert.deepEqual(first.body, { id: first.body!.id, superseded: 0, key: write.key, replayed: false, status: "accepted" }); + await call({ op: "purge", handle: first.body!.id, reason: "leaked" }); + const late = await call(write); + assert.deepEqual(late.body, { id: first.body!.id, superseded: 0, key: write.key, replayed: true, status: "rejected" }); + assert.equal((await row(first.body!.id)).content, "[purged]"); + const plain = await call({ op: "correct", subject: "other", content: "x", project: "p" }); + assert.deepEqual(Object.keys(plain.body!).sort(), ["id", "superseded"]); +}); + +test("capabilities says this Worker honours idempotency keys", async () => { + const { call } = await setup(); + assert.deepEqual((await call({ op: "capabilities" })).body, { idempotent_writes: true }); +});