From 5cd2255c6c46cfc6144e0a99be68ac421b69d7dd Mon Sep 17 00:00:00 2001 From: vu1n <2105991+vu1n@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:04:03 +0000 Subject: [PATCH 1/6] offline sync: resend queued writes under an idempotency key Operator remember/correct take an optional key. The Worker stores it in operator_receipts in the claim's own batch and answers a repeat with the first claim, so kypp sync now resends a write after any failure and it still lands once. Rows sent before keys keep the manual --drop/--resend. --- docs/features/hosted-worker.md | 4 + docs/features/offline.md | 15 ++-- kypp/offline.py | 43 +++++---- tests/test_offline.py | 93 ++++++++++++++------ worker/migrations/0012_operator_receipts.sql | 5 ++ worker/src/operator.ts | 14 ++- worker/src/store.ts | 34 ++++++- worker/test/operator.test.ts | 48 ++++++++++ 8 files changed, 200 insertions(+), 56 deletions(-) create mode 100644 worker/migrations/0012_operator_receipts.sql diff --git a/docs/features/hosted-worker.md b/docs/features/hosted-worker.md index 2cf9ec4..7240fa0 100644 --- a/docs/features/hosted-worker.md +++ b/docs/features/hosted-worker.md @@ -43,6 +43,10 @@ TypeSafe API's shape, for GitHub Actions OIDC tokens (`worker/src/oidc.ts`) and `defrag_log`. `markVerified` mirrors `MemoryStore.mark_verified`: change one, change both. - The operator `snapshot` op is `kypp sync`'s read-only pull: recall's visibility with every candidate (`visible(..., "all")`), paged by id. A field the offline cache needs must be in `CLAIM_COLUMNS`. +- Operator `remember`/`correct` take an optional idempotency `key`. Its `operator_receipts` row goes + in the claim's own batch, so a repeat (or a racing send whose batch lost on the primary key) returns + the first claim with `replayed: true`; the same key with a different write is refused. Receipts are + kept for good: a write queued offline may be resent at any later sync. - Hosted operator `correct` is an explicit replacement in the existing `memory_claims` store, not a rank contest. Insert, FTS/pointers, exact-group supersession and audit share one D1 batch; the returned superseded count comes from that transaction. Equal clocks, query order and lower diff --git a/docs/features/offline.md b/docs/features/offline.md index 214cfa8..7e037b1 100644 --- a/docs/features/offline.md +++ b/docs/features/offline.md @@ -20,12 +20,15 @@ operator's view of the project (`POST /v1/operator` `snapshot`, 500 claims a pag - Only `remote.Unreachable` (no connection was made) falls back to the cache or queue. urllib wraps a failure while sending in `URLError` too, so `remote.py` records whether `connect()` finished (`_conn.made`); after it, a failure is ambiguous and never falls back. -- A queued write is marked sent before its request and deleted only after a 200. Only `Unreachable` - unmarks it. Anything else (an error answer, a timeout, a dropped connection) may follow a committed - write: the Worker answers 400 for any failure, even after a commit, and operator `remember` has no - idempotency key. A marked write blocks the push until the operator checks and runs `kypp sync - --drop N` or `--resend N`; it is never re-sent on its own (a purge must not be undone by a replay). - The pull still runs, so a stuck write doesn't freeze the copy. +- Each queued write gets an idempotency key, saved on disk before its first send and sent again on + every retry. The Worker stores the key in `operator_receipts` in the claim's own batch and answers a + repeat with the first claim, so a write is resent automatically after any failure (an error answer, + a timeout, a lost reply) and still lands once; a purged claim is never written back by a late + resend. A write leaves the queue only after a 200 and blocks the writes behind it until then; a write + the Worker keeps refusing stays until `kypp sync --drop N`. The pull still runs, so a stuck write + doesn't freeze the copy. +- A row already sent by a client from before keys has none, so it may or may not have landed: it blocks + the push until the operator checks and runs `--drop N` or `--resend N`. - The cache is a copy, never an authority: nothing in `offline.py` accepts, promotes or supersedes. The Worker applies its own rules when a queued write arrives. - `correct`, `reject` and `purge` are refused offline (`shell._online`): they replace or retire memory diff --git a/kypp/offline.py b/kypp/offline.py index 709a972..6733ca7 100644 --- a/kypp/offline.py +++ b/kypp/offline.py @@ -13,6 +13,7 @@ import json import os import re +import secrets import sqlite3 import sys import time @@ -33,7 +34,7 @@ CREATE TABLE IF NOT EXISTS forgotten (url TEXT NOT NULL, id TEXT NOT NULL, purged INTEGER NOT NULL, at REAL NOT NULL, PRIMARY KEY (url, id)); CREATE TABLE IF NOT EXISTS outbox (seq INTEGER PRIMARY KEY AUTOINCREMENT, url TEXT NOT NULL, project TEXT NOT NULL, - op TEXT NOT NULL, args TEXT NOT NULL, queued_at TEXT NOT NULL, sent_at TEXT); + op TEXT NOT NULL, args TEXT NOT NULL, queued_at TEXT NOT NULL, sent_at TEXT, key TEXT); """ @@ -56,6 +57,9 @@ def _db() -> sqlite3.Connection: os.umask(old) os.chmod(path, 0o600) db.executescript(_SCHEMA) + # A cache made before writes carried keys gains the column; push gives each row its key. + if "key" not in {r[1] for r in db.execute("PRAGMA table_info(outbox)")}: + db.execute("ALTER TABLE outbox ADD COLUMN key TEXT") return db @@ -193,31 +197,35 @@ def forget(claim_id: str, purged: bool) -> None: def push(db: sqlite3.Connection) -> int: - """Send queued writes in order. Each is marked sent before its request and leaves the queue only - after a 200. Only a request that never got a connection is unmarked again. Any other failure (an - error answer, a timeout, a dropped connection) may follow a committed write: the Worker answers 400 - for any failure, even after a commit, and has no idempotency key. So a marked write is never sent - again on its own; the operator checks the hosted store and runs `--drop N` or `--resend N`.""" + """Send queued writes in order, each under its own idempotency key, so the Worker + applies it at most once however often it is sent. A write leaves the queue only after a 200; any + other answer (or none) leaves it, and the writes after it, for the next `kypp sync` to send again. + Only a row from before keys that was already sent can't be resent safely: the operator decides it.""" sent = 0 - for seq, op, raw, sent_at in db.execute( - "SELECT seq, op, args, sent_at FROM outbox WHERE url = ? ORDER BY seq", (_url(),)).fetchall(): + for seq, op, raw, sent_at, key in db.execute( + "SELECT seq, op, args, sent_at, key FROM outbox WHERE url = ? ORDER BY seq", (_url(),)).fetchall(): if op not in _QUEUEABLE: raise SystemExit(f"kypp: queued write #{seq} is `{op}`, which is never queued; " f"drop it with `kypp sync --drop {seq}`") stored = json.loads(raw) args = {k: stored.get(k) for k in _QUEUEABLE[op]} - if sent_at: - raise SystemExit(_unsure(seq, args, f"it was sent at {sent_at} and no 200 came back")) + # The key is fixed, on disk, before the row's first send and reused on every later one. + if not key: + if sent_at: + raise SystemExit(_unsure(seq, args, f"it was sent at {sent_at}, before writes carried keys, " + "and no 200 came back")) + key = secrets.token_hex(16) with db: - db.execute("UPDATE outbox SET sent_at = ? WHERE seq = ?", (_now(), seq)) + db.execute("UPDATE outbox SET sent_at = ?, key = ? WHERE seq = ?", (_now(), key, seq)) try: - remote.call(op, **args) + remote.call(op, key=key, **args) except remote.Unreachable: - with db: # no connection was made, so nothing was applied - db.execute("UPDATE outbox SET sent_at = NULL WHERE seq = ?", (seq,)) - raise + raise # still offline: sync_main reports it as such except SystemExit as e: - raise SystemExit(f"{e.code}\n" + _unsure(seq, args, "the Worker didn't answer 200")) from None + raise SystemExit(f"{e.code}\nkypp: queued write #{seq} ({args.get('subject')!r}) and the writes after " + f"it stay queued; the next `kypp sync` sends it again under the same key, so it lands " + f"at most once. If the Worker refused it, fix the cause or run " + f"`kypp sync --drop {seq}`.") from None with db: db.execute("DELETE FROM outbox WHERE seq = ?", (seq,)) sent += 1 @@ -252,7 +260,8 @@ def sync_main(): ap.add_argument("--drop", type=int, action="append", default=[], metavar="N", help="discard queued write #N without sending it (it reached the store, or it was refused)") ap.add_argument("--resend", type=int, action="append", default=[], metavar="N", - help="send queued write #N again: you checked that the hosted store doesn't have it") + help="send queued write #N again (one queued before writes carried keys): you checked that the " + "hosted store doesn't have it") args = ap.parse_args() if not remote.remote_url(): raise SystemExit("kypp sync: set KYPP_URL; the local store needs no sync") diff --git a/tests/test_offline.py b/tests/test_offline.py index bdeb7d0..c3e0318 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -28,6 +28,8 @@ class _Worker(http.server.BaseHTTPRequestHandler): refuse: set = set() # ops answered 400 page = 2 claim_reply: dict = {} + applied: dict = {} # idempotency key -> claim id, as the Worker's operator_receipts + def do_POST(self): body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) @@ -41,7 +43,9 @@ def do_POST(self): got = rest[:self.page] return self._send(200, {"claims": got, "next": got[-1]["id"] if len(rest) > self.page else None}) if body["op"] == "remember": - return self._send(200, {"id": "f" * 32, "status": "candidate"}) + replayed = body.get("key") in self.applied + cid = self.applied.setdefault(body.get("key"), f"{len(self.applied):032x}") + return self._send(200, {"id": cid, "status": "candidate", "key": body.get("key"), "replayed": replayed}) if body["op"] in ("purge", "reject"): return self._send(200, self.claim_reply) return self._send(200, {"text": "online answer"}) @@ -77,7 +81,7 @@ def _run(main, *argv) -> tuple[str, str]: class OfflineTest(unittest.TestCase): def setUp(self): _Worker.claims = [_claim(1), _claim(2, status="candidate"), _claim(3, type="pitfall")] - _Worker.seen, _Worker.refuse, _Worker.page = [], set(), 2 + _Worker.seen, _Worker.refuse, _Worker.page, _Worker.applied = [], set(), 2, {} self.srv = http.server.HTTPServer(("127.0.0.1", 0), _Worker) threading.Thread(target=self.srv.serve_forever, daemon=True).start() self.addCleanup(self.srv.server_close) @@ -178,57 +182,94 @@ def test_queued_writes_leave_the_queue_only_once_the_worker_answers(self): out, _ = self.sync() self.assertIn("sent 0 queued write(s)", out) - def test_a_write_without_a_200_is_never_sent_again_on_its_own(self): - # An error answer, a gateway error or a lost reply may follow a committed write (the Worker answers - # 400 even after a commit, and has no idempotency key), so only the operator decides. + def test_a_write_without_a_200_is_sent_again_under_its_key_and_lands_once(self): + # A refusal, a gateway error or a lost reply after a commit: the next sync resends the same key, + # and the Worker returns the first claim instead of inserting another. for failure in ("refused", "lost"): with self.subTest(failure=failure): self.setUp() dead = _dead_url() self.offline_copy(dead) with self.env_at(dead): - _run(shell.remember_main, "leaked", "the token is abc") + _run(shell.remember_main, "a", "first") _run(shell.remember_main, "b", "second") self.back_online() - orig = remote.call + orig, fails = remote.call, [1] - def fail(op, **args): - if op == "remember": - _Worker.seen.append({"op": op, **args}) + def flaky(op, **args): + if op == "remember" and fails: + fails.pop() + if failure == "lost": + orig(op, **args) # committed; the answer never arrives raise SystemExit("kypp: answered 400" if failure == "refused" else "kypp: timed out") return orig(op, **args) - with self.env_at(self.url), mock.patch.object(remote, "call", fail): + with self.env_at(self.url), mock.patch.object(remote, "call", flaky): with self.assertRaises(SystemExit) as e: _run(offline.sync_main) - self.assertIn("may already be in the hosted store", str(e.exception.code)) + self.assertIn("sends it again under the same key", str(e.exception.code)) self.assertIn("cached", str(e.exception.code)) # the copy still refreshed - for _ in range(2): # e.g. the operator purged it meanwhile: still not re-sent - with self.assertRaises(SystemExit) as e: - self.sync() - self.assertIn("--resend 1", str(e.exception.code)) - self.assertEqual(len([b for b in _Worker.seen if b["op"] == "remember"]), 1) with self.env_at(self.url): - out, _ = _run(offline.sync_main, "--drop", "1") - self.assertIn("dropped queued write #1", out) - self.assertIn("sent 1 queued write(s)", out) # "b" goes once the operator decided - - def test_resend_sends_a_write_the_operator_checked(self): + self.assertEqual(len(offline._queued(offline._db(), "p")), 2) # "b" waits behind "a" + out, _ = self.sync() + self.assertIn("sent 2 queued write(s)", out) + sent = [b for b in _Worker.seen if b["op"] == "remember"] + self.assertEqual([b["subject"] for b in sent], ["a", "a", "b"] if failure == "lost" else ["a", "b"]) + self.assertEqual(len({b["key"] for b in sent}), 2) # a resend reuses its key + self.assertEqual(len(_Worker.applied), 2) + + def test_a_row_sent_before_keys_waits_for_the_operator(self): dead = _dead_url() self.offline_copy(dead) with self.env_at(dead): _run(shell.remember_main, "a", "first") + _run(shell.remember_main, "b", "second") self.back_online() with self.env_at(self.url): db = offline._db() - db.execute("UPDATE outbox SET sent_at = 'earlier'") + db.execute("UPDATE outbox SET key = NULL") # as a cache from before keys + db.execute("UPDATE outbox SET sent_at = 'earlier' WHERE seq = 1") db.commit() - with self.assertRaises(SystemExit): - self.sync() + for _ in range(2): + with self.assertRaises(SystemExit) as e: + self.sync() + self.assertIn("before writes carried keys", str(e.exception.code)) + self.assertIn("--resend 1", str(e.exception.code)) + self.assertEqual([b for b in _Worker.seen if b["op"] == "remember"], []) with self.env_at(self.url): out, _ = _run(offline.sync_main, "--resend", "1") + self.assertIn("sent 2 queued write(s)", out) # an unsent row from before keys gets one + self.assertTrue(all(b.get("key") for b in _Worker.seen if b["op"] == "remember")) + + def test_a_cache_from_before_keys_gains_the_column(self): + import sqlite3 + path = self.env["KYPP_CACHE_DB"] + old = sqlite3.connect(path) + old.execute("CREATE TABLE outbox (seq INTEGER PRIMARY KEY AUTOINCREMENT, url TEXT NOT NULL, project TEXT " + "NOT NULL, op TEXT NOT NULL, args TEXT NOT NULL, queued_at TEXT NOT NULL, sent_at TEXT)") + old.execute("INSERT INTO outbox(url, project, op, args, queued_at) VALUES(?, 'p', 'remember', ?, 'x')", + (self.url, json.dumps({"subject": "old", "content": "row", "project": "p"}))) + old.commit() + old.close() + out, _ = self.sync() + self.assertIn("sent 1 queued write(s)", out) + + def test_drop_discards_a_write_the_worker_keeps_refusing(self): + dead = _dead_url() + self.offline_copy(dead) + with self.env_at(dead): + _run(shell.remember_main, "a", "first") + _run(shell.remember_main, "b", "second") + self.back_online() + _Worker.refuse = {"remember"} + with self.assertRaises(SystemExit) as e: + self.sync() + self.assertIn("--drop 1", str(e.exception.code)) + _Worker.refuse = set() + with self.env_at(self.url): + out, _ = _run(offline.sync_main, "--drop", "1") + self.assertIn("dropped queued write #1", out) self.assertIn("sent 1 queued write(s)", out) - self.assertEqual(len([b for b in _Worker.seen if b["op"] == "remember"]), 1) def test_a_tampered_queue_row_never_becomes_another_operator_write(self): dead = _dead_url() diff --git a/worker/migrations/0012_operator_receipts.sql b/worker/migrations/0012_operator_receipts.sql new file mode 100644 index 0000000..2903c13 --- /dev/null +++ b/worker/migrations/0012_operator_receipts.sql @@ -0,0 +1,5 @@ +-- One row per operator write sent with an idempotency key: a resend of that key returns this claim +-- instead of inserting another. Kept for good, so a write queued offline can be resent at any later sync. +CREATE TABLE IF NOT EXISTS operator_receipts ( + key TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, claim_id TEXT NOT NULL, created_at TEXT NOT NULL +); diff --git a/worker/src/operator.ts b/worker/src/operator.ts index d3bbea7..7c7323d 100644 --- a/worker/src/operator.ts +++ b/worker/src/operator.ts @@ -15,6 +15,8 @@ const OPERATOR_USER = "owner"; const SNAPSHOT_PAGE = 500; // A repo's bare name, as `resolveOrigin` matches the last segment of an origin like `owner/repo`. const PROJECT_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$/; +// An idempotency key the client makes once per write and sends again on every retry of it. +const WRITE_KEY = /^[A-Za-z0-9_-]{16,128}$/; type Args = Record; export interface OperatorOptions { s1?: S1Client | null; policy?: Policy } @@ -63,13 +65,17 @@ async function run(store: D1Store, a: Args, opts: OperatorOptions): Promise { + // With a key, the write lands at most once: the receipt is inserted in the claim's own batch, so a + // resend of the key (after a lost answer) returns the first claim instead of inserting another. + async operatorClaim(c: ClaimInput, op: "remember" | "correct", verify: string | null, key: string | null = null): Promise<{ id: string; superseded: number; replayed: boolean }> { if (!TYPES.includes(c.type)) throw new Error(`bad type ${c.type}`); if (!SCOPES.includes(c.scope) || c.scope === "group" || c.scope === "room") throw new Error(`bad scope ${c.scope}`); if (c.scope === "project" && !c.project) throw new Error("a project-scoped operator claim needs a known project"); @@ -246,7 +248,20 @@ export class D1Store { const project = c.scope === "project" ? c.project : null; const confidence = Math.min(1, Math.max(0, Number(c.confidence) || 0)); const codeRefs = cleanCodeRefs(c.codeRefs); + const fingerprint = key ? await sha256(JSON.stringify([op, c.type, c.subject, c.content, c.scope, project, + c.project, confidence, !!c.accept, verify, c.agent, c.user])) : ""; + const replay = async () => { + const r = await this.db.prepare("SELECT fingerprint, claim_id FROM operator_receipts WHERE key = ?").bind(key).first<{ fingerprint: string; claim_id: string }>(); + if (r && r.fingerprint !== fingerprint) throw new Error(`key ${JSON.stringify(key)} was already used for a different write`); + return r ? { id: r.claim_id, superseded: 0, replayed: true } : null; + }; + if (key) { + const prior = await replay(); + if (prior) return prior; + } const statements = [ + ...(key ? [this.db.prepare("INSERT INTO operator_receipts(key, fingerprint, claim_id, created_at) VALUES(?,?,?,?)") + .bind(key, fingerprint, id, ts)] : []), this.db.prepare( "INSERT INTO memory_claims(id,type,subject,content,scope,project,origin,session,agent,user,status,authority,confidence,source_ids,code_refs,metadata,verify,created_at,updated_at)" + " VALUES(?,?,?,?,?,?,?,NULL,?,?,?,?,?,'[]',?,'{}',?,?,?)", @@ -271,8 +286,16 @@ export class D1Store { this.db.prepare(`UPDATE memory_claims SET status='superseded', updated_at=? WHERE ${prior}`).bind(ts, ...key), ); } - const results = await this.db.batch(statements); - return { id, superseded: op === "correct" ? results[results.length - 1].meta.changes : 0 }; + let results; + try { + results = await this.db.batch(statements); + } catch (e) { + // A concurrent send of the same key committed first: the receipt's primary key rolled this batch back. + const prior = key ? await replay() : null; + if (prior) return prior; + throw e; + } + return { id, superseded: op === "correct" ? results[results.length - 1].meta.changes : 0, replayed: false }; } // Every claim the operator's `kypp verify` re-checks in this project: any status, so a rejected @@ -737,3 +760,8 @@ export class D1Store { return { applied, stale }; } } + +async function sha256(value: string): Promise { + const bytes = new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value))); + return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join(""); +} diff --git a/worker/test/operator.test.ts b/worker/test/operator.test.ts index 49d01d0..0e221bb 100644 --- a/worker/test/operator.test.ts +++ b/worker/test/operator.test.ts @@ -287,3 +287,51 @@ test("snapshot pages the operator's live view of one project by id, without reje assert.deepEqual(new Set([...first.body!.claims, ...rest.body!.claims].map((c: any) => c.status)), new Set(["accepted", "candidate"])); assert.equal((await call({ op: "snapshot", project: "nope" })).status, 400); }); + +test("a remember resent with its key lands once, and a key reused for another write is refused", async () => { + const { db, call, row } = await setup(); + const write = { op: "remember", subject: "deploy", content: "ship on green", project: "p", key: "k".repeat(24) }; + const first = await call(write); + assert.equal(first.status, 200); + assert.equal(first.body!.replayed, false); + const again = await call(write); + assert.equal(again.status, 200); + assert.deepEqual(again.body, { id: first.body!.id, status: "candidate", key: write.key, replayed: true }); + const n = async () => (await db.prepare("SELECT count(*) AS n FROM memory_claims WHERE subject = 'deploy'").first() as any).n; + assert.equal(await n(), 1); + const other = await call({ ...write, content: "ship any time" }); + assert.equal(other.status, 400); + assert.match(other.body!.error, /already used for a different write/); + assert.equal(await n(), 1); + // A purged claim is not written back by a late resend of its key. + await call({ op: "purge", handle: first.body!.id, reason: "leaked" }); + const late = await call(write); + assert.equal(late.body!.replayed, true); + assert.equal(late.body!.status, "rejected"); + assert.equal(await n(), 0); + assert.equal((await row(first.body!.id)).content, "[purged]"); +}); + +test("a send of a key that loses the race to another send of it returns the winner's claim", async () => { + const db = memoryD1(); + const input = { type: "fact" as const, subject: "race", content: "one copy", scope: "global" as const, project: null, + confidence: 0.7, sourceIds: [], codeRefs: [], accept: false, agent: "kypp-cli", user: "owner" }; + const key = "r".repeat(24); + let winner = ""; + // The other send commits after this one checked for a receipt and before its batch runs. + const racing = new Proxy(db, { get: (t, p) => p !== "batch" ? (t as any)[p] : async (stmts: unknown[]) => { + if (!winner) winner = (await new D1Store(db).operatorClaim(input, "remember", null, key)).id; + return t.batch(stmts as any); + } }); + const loser = await new D1Store(racing).operatorClaim(input, "remember", null, key); + assert.deepEqual(loser, { id: winner, superseded: 0, replayed: true }); + assert.equal((await db.prepare("SELECT count(*) AS n FROM memory_claims WHERE subject = 'race'").first() as any).n, 1); +}); + +test("a malformed key is refused before anything is written, and no key keeps the old answer", async () => { + const { call } = await setup(); + const bad = await call({ op: "remember", subject: "s", content: "c", project: "p", key: "short" }); + assert.equal(bad.status, 400); + const plain = await call({ op: "remember", subject: "s", content: "c", project: "p" }); + assert.deepEqual(Object.keys(plain.body!).sort(), ["id", "status"]); +}); From f66cb97c5d33c3e65a52da072a4b28ba802c1038 Mon Sep 17 00:00:00 2001 From: vu1n <2105991+vu1n@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:08:53 +0000 Subject: [PATCH 2/6] operator: a keyed correct reports the claim's current status too --- docs/features/hosted-worker.md | 2 +- worker/src/operator.ts | 2 +- worker/test/operator.test.ts | 13 +++++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/features/hosted-worker.md b/docs/features/hosted-worker.md index 7240fa0..e309d02 100644 --- a/docs/features/hosted-worker.md +++ b/docs/features/hosted-worker.md @@ -45,7 +45,7 @@ TypeSafe API's shape, for GitHub Actions OIDC tokens (`worker/src/oidc.ts`) and candidate (`visible(..., "all")`), paged by id. A field the offline cache needs must be in `CLAIM_COLUMNS`. - Operator `remember`/`correct` take an optional idempotency `key`. Its `operator_receipts` row goes in the claim's own batch, so a repeat (or a racing send whose batch lost on the primary key) returns - the first claim with `replayed: true`; the same key with a different write is refused. Receipts are + the first claim with `replayed: true` and its current `status`; the same key with a different write is refused. Receipts are kept for good: a write queued offline may be resent at any later sync. - Hosted operator `correct` is an explicit replacement in the existing `memory_claims` store, not a rank contest. Insert, FTS/pointers, exact-group supersession and audit share one D1 batch; diff --git a/worker/src/operator.ts b/worker/src/operator.ts index 7c7323d..c3b605d 100644 --- a/worker/src/operator.ts +++ b/worker/src/operator.ts @@ -75,7 +75,7 @@ async function run(store: D1Store, a: Args, opts: OperatorOptions): Promise { + const { call, row } = await setup(); + const write = { op: "correct", subject: "deploy", content: "ship on green", project: "p", key: "c".repeat(24) }; + const first = await call(write); + assert.deepEqual(first.body, { id: first.body!.id, superseded: 0, key: write.key, replayed: false, status: "accepted" }); + await call({ op: "purge", handle: first.body!.id, reason: "leaked" }); + const late = await call(write); + assert.deepEqual(late.body, { id: first.body!.id, superseded: 0, key: write.key, replayed: true, status: "rejected" }); + assert.equal((await row(first.body!.id)).content, "[purged]"); + const plain = await call({ op: "correct", subject: "other", content: "x", project: "p" }); + assert.deepEqual(Object.keys(plain.body!).sort(), ["id", "superseded"]); +}); From 033569fed1f5d8a006b9b127bcbf2e2ce6fb3089 Mon Sep 17 00:00:00 2001 From: vu1n <2105991+vu1n@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:15:26 +0000 Subject: [PATCH 3/6] offline sync: resend automatically only to a Worker that says it honours keys --- docs/features/hosted-worker.md | 3 +- docs/features/offline.md | 21 ++++++------ kypp/offline.py | 48 +++++++++++++++++++++------- tests/test_offline.py | 58 ++++++++++++++++++++++++++++++++-- worker/src/operator.ts | 3 ++ worker/test/operator.test.ts | 5 +++ 6 files changed, 113 insertions(+), 25 deletions(-) diff --git a/docs/features/hosted-worker.md b/docs/features/hosted-worker.md index e309d02..5eaf12e 100644 --- a/docs/features/hosted-worker.md +++ b/docs/features/hosted-worker.md @@ -46,7 +46,8 @@ TypeSafe API's shape, for GitHub Actions OIDC tokens (`worker/src/oidc.ts`) and - Operator `remember`/`correct` take an optional idempotency `key`. Its `operator_receipts` row goes in the claim's own batch, so a repeat (or a racing send whose batch lost on the primary key) returns the first claim with `replayed: true` and its current `status`; the same key with a different write is refused. Receipts are - kept for good: a write queued offline may be resent at any later sync. + kept for good: a write queued offline may be resent at any later sync. The `capabilities` op + (`idempotent_writes: true`) is how the CLI tells this Worker from one that ignores `key`. - Hosted operator `correct` is an explicit replacement in the existing `memory_claims` store, not a rank contest. Insert, FTS/pointers, exact-group supersession and audit share one D1 batch; the returned superseded count comes from that transaction. Equal clocks, query order and lower diff --git a/docs/features/offline.md b/docs/features/offline.md index 7e037b1..0f6c45a 100644 --- a/docs/features/offline.md +++ b/docs/features/offline.md @@ -20,15 +20,18 @@ operator's view of the project (`POST /v1/operator` `snapshot`, 500 claims a pag - Only `remote.Unreachable` (no connection was made) falls back to the cache or queue. urllib wraps a failure while sending in `URLError` too, so `remote.py` records whether `connect()` finished (`_conn.made`); after it, a failure is ambiguous and never falls back. -- Each queued write gets an idempotency key, saved on disk before its first send and sent again on - every retry. The Worker stores the key in `operator_receipts` in the claim's own batch and answers a - repeat with the first claim, so a write is resent automatically after any failure (an error answer, - a timeout, a lost reply) and still lands once; a purged claim is never written back by a late - resend. A write leaves the queue only after a 200 and blocks the writes behind it until then; a write - the Worker keeps refusing stays until `kypp sync --drop N`. The pull still runs, so a stuck write - doesn't freeze the copy. -- A row already sent by a client from before keys has none, so it may or may not have landed: it blocks - the push until the operator checks and runs `--drop N` or `--resend N`. +- Before a push, `kypp sync` asks the Worker for `capabilities`. Against one that honours keys, each + queued write gets an idempotency key, saved on disk before its first send and sent again on every + retry. The Worker stores the key in `operator_receipts` in the claim's own batch and answers a repeat + with the first claim, so a write is resent automatically after any failure (an error answer, a + timeout, a lost reply) and still lands once; a purged claim is never written back by a late resend. + A write leaves the queue only after a 200 and blocks the writes behind it until then; a write the + Worker keeps refusing stays until `kypp sync --drop N`. The pull still runs, so a stuck write doesn't + freeze the copy. +- A Worker deployed before keys answers `unknown op` to `capabilities` and ignores `key`, so the write + goes without one. A row sent without a key (to such a Worker, or by an older CLI), or a keyed row the + Worker no longer honours, may or may not have landed: it blocks the push until the operator checks and + runs `--drop N` or `--resend N`. A send that never connected is unmarked and goes again as new. - The cache is a copy, never an authority: nothing in `offline.py` accepts, promotes or supersedes. The Worker applies its own rules when a queued write arrives. - `correct`, `reject` and `purge` are refused offline (`shell._online`): they replace or retire memory diff --git a/kypp/offline.py b/kypp/offline.py index 6733ca7..1c2c984 100644 --- a/kypp/offline.py +++ b/kypp/offline.py @@ -196,12 +196,25 @@ def forget(claim_id: str, purged: bool) -> None: db.execute("DELETE FROM claims WHERE url = ? AND id = ?", (_url(), claim_id)) +def _keyed() -> bool: + """Whether the Worker honours idempotency keys. One deployed before them ignores `key`.""" + try: + return remote.call("capabilities").get("idempotent_writes") is True + except remote.Unreachable: + raise + except SystemExit as e: + if "unknown op" in str(e.code): + return False + raise + + def push(db: sqlite3.Connection) -> int: - """Send queued writes in order, each under its own idempotency key, so the Worker - applies it at most once however often it is sent. A write leaves the queue only after a 200; any - other answer (or none) leaves it, and the writes after it, for the next `kypp sync` to send again. - Only a row from before keys that was already sent can't be resent safely: the operator decides it.""" - sent = 0 + """Send queued writes in order. Against a Worker that honours keys, each is sent under its own + idempotency key, so it is applied at most once however often it is sent: a write leaves the queue + only after a 200, and any other answer (or none) leaves it, and the writes after it, for the next + `kypp sync` to send again. A row sent without a key (to an older Worker, or by an older CLI) may or + may not have landed, so it is never resent on its own: the operator decides it.""" + sent, keyed = 0, None for seq, op, raw, sent_at, key in db.execute( "SELECT seq, op, args, sent_at, key FROM outbox WHERE url = ? ORDER BY seq", (_url(),)).fetchall(): if op not in _QUEUEABLE: @@ -209,19 +222,30 @@ def push(db: sqlite3.Connection) -> int: f"drop it with `kypp sync --drop {seq}`") stored = json.loads(raw) args = {k: stored.get(k) for k in _QUEUEABLE[op]} - # The key is fixed, on disk, before the row's first send and reused on every later one. + # A row's key is fixed, on disk, before its first send, and only once the Worker said it honours + # keys; a row sent without one is never resent automatically. + if not key and sent_at: + raise SystemExit(_unsure(seq, args, f"it was sent at {sent_at} without an idempotency key " + "and no 200 came back")) + if keyed is None: + keyed = _keyed() + if key and sent_at and not keyed: + raise SystemExit(_unsure(seq, args, f"it was sent at {sent_at} and the Worker no longer honours " + "idempotency keys")) if not key: - if sent_at: - raise SystemExit(_unsure(seq, args, f"it was sent at {sent_at}, before writes carried keys, " - "and no 200 came back")) - key = secrets.token_hex(16) + key = secrets.token_hex(16) if keyed else None with db: db.execute("UPDATE outbox SET sent_at = ?, key = ? WHERE seq = ?", (_now(), key, seq)) try: remote.call(op, key=key, **args) except remote.Unreachable: - raise # still offline: sync_main reports it as such + with db: # no connection was made, so nothing was applied + db.execute("UPDATE outbox SET sent_at = NULL WHERE seq = ?", (seq,)) + raise except SystemExit as e: + if not key: + raise SystemExit(f"{e.code}\n" + _unsure(seq, args, "the Worker (which predates idempotency keys) " + "didn't answer 200")) from None raise SystemExit(f"{e.code}\nkypp: queued write #{seq} ({args.get('subject')!r}) and the writes after " f"it stay queued; the next `kypp sync` sends it again under the same key, so it lands " f"at most once. If the Worker refused it, fix the cause or run " @@ -260,7 +284,7 @@ def sync_main(): ap.add_argument("--drop", type=int, action="append", default=[], metavar="N", help="discard queued write #N without sending it (it reached the store, or it was refused)") ap.add_argument("--resend", type=int, action="append", default=[], metavar="N", - help="send queued write #N again (one queued before writes carried keys): you checked that the " + help="send queued write #N again (one sent without an idempotency key): you checked that the " "hosted store doesn't have it") args = ap.parse_args() if not remote.remote_url(): diff --git a/tests/test_offline.py b/tests/test_offline.py index c3e0318..81a313d 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -29,6 +29,7 @@ class _Worker(http.server.BaseHTTPRequestHandler): page = 2 claim_reply: dict = {} applied: dict = {} # idempotency key -> claim id, as the Worker's operator_receipts + old = False # a Worker deployed before idempotency keys: no `capabilities`, `key` ignored def do_POST(self): @@ -42,6 +43,13 @@ def do_POST(self): rest = [c for c in self.claims if c["id"] > body.get("after", "")] got = rest[:self.page] return self._send(200, {"claims": got, "next": got[-1]["id"] if len(rest) > self.page else None}) + if body["op"] == "capabilities": + if self.old: + return self._send(400, {"error": 'unknown op "capabilities"'}) + return self._send(200, {"idempotent_writes": True}) + if body["op"] == "remember" and self.old: + self.applied[len(self.applied)] = body["subject"] + return self._send(200, {"id": "e" * 32, "status": "candidate"}) if body["op"] == "remember": replayed = body.get("key") in self.applied cid = self.applied.setdefault(body.get("key"), f"{len(self.applied):032x}") @@ -81,7 +89,7 @@ def _run(main, *argv) -> tuple[str, str]: class OfflineTest(unittest.TestCase): def setUp(self): _Worker.claims = [_claim(1), _claim(2, status="candidate"), _claim(3, type="pitfall")] - _Worker.seen, _Worker.refuse, _Worker.page, _Worker.applied = [], set(), 2, {} + _Worker.seen, _Worker.refuse, _Worker.page, _Worker.applied, _Worker.old = [], set(), 2, {}, False self.srv = http.server.HTTPServer(("127.0.0.1", 0), _Worker) threading.Thread(target=self.srv.serve_forever, daemon=True).start() self.addCleanup(self.srv.server_close) @@ -233,7 +241,7 @@ def test_a_row_sent_before_keys_waits_for_the_operator(self): for _ in range(2): with self.assertRaises(SystemExit) as e: self.sync() - self.assertIn("before writes carried keys", str(e.exception.code)) + self.assertIn("without an idempotency key", str(e.exception.code)) self.assertIn("--resend 1", str(e.exception.code)) self.assertEqual([b for b in _Worker.seen if b["op"] == "remember"], []) with self.env_at(self.url): @@ -241,6 +249,50 @@ def test_a_row_sent_before_keys_waits_for_the_operator(self): self.assertIn("sent 2 queued write(s)", out) # an unsent row from before keys gets one self.assertTrue(all(b.get("key") for b in _Worker.seen if b["op"] == "remember")) + def test_a_lost_reply_from_a_worker_without_keys_is_never_resent_on_its_own(self): + # A Worker deployed before keys ignores `key`; a resend after a committed write would duplicate it. + _Worker.old = True + dead = _dead_url() + self.offline_copy(dead) + with self.env_at(dead): + _run(shell.remember_main, "a", "first") + self.back_online() + orig, fails = remote.call, [1] + + def lost(op, **args): + if op == "remember" and fails: + fails.pop() + orig(op, **args) # committed; the answer never arrives + raise SystemExit("kypp: timed out") + return orig(op, **args) + + with self.env_at(self.url), mock.patch.object(remote, "call", lost): + with self.assertRaises(SystemExit) as e: + _run(offline.sync_main) + self.assertIn("predates idempotency keys", str(e.exception.code)) + _Worker.old = False # even once the Worker is upgraded, the unkeyed send stays the operator's call + for _ in range(2): + with self.assertRaises(SystemExit) as e: + self.sync() + self.assertIn("--resend 1", str(e.exception.code)) + self.assertEqual(len(_Worker.applied), 1) + self.assertNotIn("key", [k for b in _Worker.seen if b["op"] == "remember" for k in b]) + + def test_a_keyed_row_is_not_resent_to_a_worker_that_stopped_honouring_keys(self): + dead = _dead_url() + self.offline_copy(dead) + with self.env_at(dead): + _run(shell.remember_main, "a", "first") + self.back_online() + _Worker.refuse = {"remember"} + with self.assertRaises(SystemExit): + self.sync() + _Worker.refuse, _Worker.old = set(), True + with self.assertRaises(SystemExit) as e: + self.sync() + self.assertIn("no longer honours idempotency keys", str(e.exception.code)) + self.assertEqual(len([b for b in _Worker.seen if b["op"] == "remember"]), 1) + def test_a_cache_from_before_keys_gains_the_column(self): import sqlite3 path = self.env["KYPP_CACHE_DB"] @@ -286,7 +338,7 @@ def test_a_tampered_queue_row_never_becomes_another_operator_write(self): with self.assertRaises(SystemExit) as e: self.sync() self.assertIn("never queued", str(e.exception.code)) - sent = [b for b in _Worker.seen if b["op"] not in ("snapshot",)] + sent = [b for b in _Worker.seen if b["op"] not in ("snapshot", "capabilities")] self.assertEqual([b["op"] for b in sent], ["remember"]) self.assertNotIn("authority", sent[0]) diff --git a/worker/src/operator.ts b/worker/src/operator.ts index c3b605d..f926222 100644 --- a/worker/src/operator.ts +++ b/worker/src/operator.ts @@ -104,6 +104,9 @@ async function run(store: D1Store, a: Args, opts: OperatorOptions): Promise { + const { call } = await setup(); + assert.deepEqual((await call({ op: "capabilities" })).body, { idempotent_writes: true }); +}); From 390137e2521f14c6deed190313dae1cf518e004c Mon Sep 17 00:00:00 2001 From: vu1n <2105991+vu1n@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:18:31 +0000 Subject: [PATCH 4/6] offline sync: an unconnected resend keeps the row's earlier send on record --- docs/features/offline.md | 3 ++- kypp/offline.py | 4 ++-- tests/test_offline.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/docs/features/offline.md b/docs/features/offline.md index 0f6c45a..fe9f005 100644 --- a/docs/features/offline.md +++ b/docs/features/offline.md @@ -31,7 +31,8 @@ operator's view of the project (`POST /v1/operator` `snapshot`, 500 claims a pag - A Worker deployed before keys answers `unknown op` to `capabilities` and ignores `key`, so the write goes without one. A row sent without a key (to such a Worker, or by an older CLI), or a keyed row the Worker no longer honours, may or may not have landed: it blocks the push until the operator checks and - runs `--drop N` or `--resend N`. A send that never connected is unmarked and goes again as new. + runs `--drop N` or `--resend N`. A send that never connected restores the row's earlier mark (none if + it was never sent), so an earlier, maybe committed send stays on record. - The cache is a copy, never an authority: nothing in `offline.py` accepts, promotes or supersedes. The Worker applies its own rules when a queued write arrives. - `correct`, `reject` and `purge` are refused offline (`shell._online`): they replace or retire memory diff --git a/kypp/offline.py b/kypp/offline.py index 1c2c984..fa31339 100644 --- a/kypp/offline.py +++ b/kypp/offline.py @@ -239,8 +239,8 @@ def push(db: sqlite3.Connection) -> int: try: remote.call(op, key=key, **args) except remote.Unreachable: - with db: # no connection was made, so nothing was applied - db.execute("UPDATE outbox SET sent_at = NULL WHERE seq = ?", (seq,)) + with db: # this attempt never connected; an earlier send may still have landed, so keep its mark + db.execute("UPDATE outbox SET sent_at = ? WHERE seq = ?", (sent_at, seq)) raise except SystemExit as e: if not key: diff --git a/tests/test_offline.py b/tests/test_offline.py index 81a313d..996741e 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -293,6 +293,39 @@ def test_a_keyed_row_is_not_resent_to_a_worker_that_stopped_honouring_keys(self) self.assertIn("no longer honours idempotency keys", str(e.exception.code)) self.assertEqual(len([b for b in _Worker.seen if b["op"] == "remember"]), 1) + def test_a_resend_that_cannot_connect_keeps_the_earlier_send_on_record(self): + dead = _dead_url() + self.offline_copy(dead) + with self.env_at(dead): + _run(shell.remember_main, "a", "first") + self.back_online() + orig, fails = remote.call, [1] + + def lost(op, **args): + if op == "remember" and fails: + fails.pop() + orig(op, **args) # committed; the answer never arrives + raise SystemExit("kypp: timed out") + return orig(op, **args) + + with self.env_at(self.url), mock.patch.object(remote, "call", lost): + with self.assertRaises(SystemExit): + _run(offline.sync_main) + + def offline_resend(op, **args): + if op == "remember": + raise remote.Unreachable("kypp: can't reach it") + return orig(op, **args) + + with self.env_at(self.url), mock.patch.object(remote, "call", offline_resend): + with self.assertRaises(SystemExit): + _run(offline.sync_main) + _Worker.old = True # the Worker is downgraded: the row was sent once, so it stays the operator's call + with self.assertRaises(SystemExit) as e: + self.sync() + self.assertIn("no longer honours idempotency keys", str(e.exception.code)) + self.assertEqual(len(_Worker.applied), 1) + def test_a_cache_from_before_keys_gains_the_column(self): import sqlite3 path = self.env["KYPP_CACHE_DB"] From c83ad28a0e7b700b871e8b7938c1189679732beb Mon Sep 17 00:00:00 2001 From: vu1n <2105991+vu1n@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:22:31 +0000 Subject: [PATCH 5/6] offline sync: a row keeps its key only while every send of it was keyed --- docs/features/offline.md | 2 +- kypp/offline.py | 16 ++++++---------- tests/test_offline.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/docs/features/offline.md b/docs/features/offline.md index fe9f005..2dc7dc3 100644 --- a/docs/features/offline.md +++ b/docs/features/offline.md @@ -29,7 +29,7 @@ operator's view of the project (`POST /v1/operator` `snapshot`, 500 claims a pag Worker keeps refusing stays until `kypp sync --drop N`. The pull still runs, so a stuck write doesn't freeze the copy. - A Worker deployed before keys answers `unknown op` to `capabilities` and ignores `key`, so the write - goes without one. A row sent without a key (to such a Worker, or by an older CLI), or a keyed row the + goes without one, and a row's key is cleared before any such send. A row sent without a key (to such a Worker, or by an older CLI), or a keyed row the Worker no longer honours, may or may not have landed: it blocks the push until the operator checks and runs `--drop N` or `--resend N`. A send that never connected restores the row's earlier mark (none if it was never sent), so an earlier, maybe committed send stays on record. diff --git a/kypp/offline.py b/kypp/offline.py index fa31339..f69ada2 100644 --- a/kypp/offline.py +++ b/kypp/offline.py @@ -222,18 +222,14 @@ def push(db: sqlite3.Connection) -> int: f"drop it with `kypp sync --drop {seq}`") stored = json.loads(raw) args = {k: stored.get(k) for k in _QUEUEABLE[op]} - # A row's key is fixed, on disk, before its first send, and only once the Worker said it honours - # keys; a row sent without one is never resent automatically. - if not key and sent_at: - raise SystemExit(_unsure(seq, args, f"it was sent at {sent_at} without an idempotency key " - "and no 200 came back")) + # A row keeps a key only while every send of it went to a Worker that honours keys: it is fixed on + # disk before such a send and cleared before any other, so only a keyed row is resent on its own. if keyed is None: keyed = _keyed() - if key and sent_at and not keyed: - raise SystemExit(_unsure(seq, args, f"it was sent at {sent_at} and the Worker no longer honours " - "idempotency keys")) - if not key: - key = secrets.token_hex(16) if keyed else None + if sent_at and not (key and keyed): + why = "and the Worker no longer honours idempotency keys" if key else "without an idempotency key" + raise SystemExit(_unsure(seq, args, f"it was sent at {sent_at} {why} and no 200 came back")) + key = (key or secrets.token_hex(16)) if keyed else None with db: db.execute("UPDATE outbox SET sent_at = ?, key = ? WHERE seq = ?", (_now(), key, seq)) try: diff --git a/tests/test_offline.py b/tests/test_offline.py index 996741e..81f7443 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -326,6 +326,40 @@ def offline_resend(op, **args): self.assertIn("no longer honours idempotency keys", str(e.exception.code)) self.assertEqual(len(_Worker.applied), 1) + def test_a_row_sent_to_a_worker_without_keys_loses_its_key(self): + # Keyed Worker, the send can't connect; downgraded Worker, the reply is lost; upgraded again. + dead = _dead_url() + self.offline_copy(dead) + with self.env_at(dead): + _run(shell.remember_main, "a", "first") + self.back_online() + orig = remote.call + + def unreachable(op, **args): + if op == "remember": + raise remote.Unreachable("kypp: can't reach it") + return orig(op, **args) + + with self.env_at(self.url), mock.patch.object(remote, "call", unreachable): + with self.assertRaises(SystemExit): + _run(offline.sync_main) + _Worker.old = True + + def lost(op, **args): + res = orig(op, **args) + if op == "remember": + raise SystemExit("kypp: timed out") + return res + + with self.env_at(self.url), mock.patch.object(remote, "call", lost): + with self.assertRaises(SystemExit): + _run(offline.sync_main) + _Worker.old = False + with self.assertRaises(SystemExit) as e: + self.sync() + self.assertIn("without an idempotency key", str(e.exception.code)) + self.assertEqual(len(_Worker.applied), 1) + def test_a_cache_from_before_keys_gains_the_column(self): import sqlite3 path = self.env["KYPP_CACHE_DB"] From d94389408960d5fdbccfc10116ec62dfda955b44 Mon Sep 17 00:00:00 2001 From: vu1n <2105991+vu1n@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:26:32 +0000 Subject: [PATCH 6/6] offline cache: add the key column inside one write transaction --- kypp/offline.py | 13 ++++++++++--- tests/test_offline.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/kypp/offline.py b/kypp/offline.py index f69ada2..3f63ab5 100644 --- a/kypp/offline.py +++ b/kypp/offline.py @@ -57,9 +57,16 @@ def _db() -> sqlite3.Connection: os.umask(old) os.chmod(path, 0o600) db.executescript(_SCHEMA) - # A cache made before writes carried keys gains the column; push gives each row its key. - if "key" not in {r[1] for r in db.execute("PRAGMA table_info(outbox)")}: - db.execute("ALTER TABLE outbox ADD COLUMN key TEXT") + # A cache made before writes carried keys gains the column; push gives each row its key. The check + # and the ALTER share one write transaction, so two CLIs opening an old cache can't both add it. + db.execute("BEGIN IMMEDIATE") + try: + if "key" not in {r[1] for r in db.execute("PRAGMA table_info(outbox)")}: + db.execute("ALTER TABLE outbox ADD COLUMN key TEXT") + db.execute("COMMIT") + except BaseException: + db.execute("ROLLBACK") + raise return db diff --git a/tests/test_offline.py b/tests/test_offline.py index 81f7443..0376f5d 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -9,6 +9,7 @@ import sys import tempfile import threading +import time import unittest import urllib.error from unittest import mock @@ -373,6 +374,47 @@ def test_a_cache_from_before_keys_gains_the_column(self): out, _ = self.sync() self.assertIn("sent 1 queued write(s)", out) + def test_clis_opening_an_old_cache_at_once_add_the_column_once(self): + import sqlite3 + path = self.env["KYPP_CACHE_DB"] + connect = sqlite3.connect + + class Slow(sqlite3.Connection): + # Widens the gap between reading the schema and changing it, where a second CLI could step in. + def execute(self, sql, *a): + cur = super().execute(sql, *a) + if sql.startswith("PRAGMA table_info"): + time.sleep(0.05) + return cur + + def slow(*a, **kw): + return connect(*a, factory=Slow, **kw) + + for round_ in range(3): + with contextlib.suppress(FileNotFoundError): + os.remove(path) + old = sqlite3.connect(path) + old.execute("CREATE TABLE outbox (seq INTEGER PRIMARY KEY AUTOINCREMENT, url TEXT NOT NULL, project " + "TEXT NOT NULL, op TEXT NOT NULL, args TEXT NOT NULL, queued_at TEXT NOT NULL, sent_at TEXT)") + old.commit() + old.close() + gate, errors = threading.Barrier(6), [] + + def open_it(): + gate.wait() + try: + offline._db().close() + except Exception as e: # noqa: BLE001 - the test reports any failure + errors.append(e) + + with mock.patch.dict(os.environ, self.env), mock.patch.object(offline.sqlite3, "connect", slow): + threads = [threading.Thread(target=open_it) for _ in range(6)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertEqual(errors, [], f"round {round_}") + def test_drop_discards_a_write_the_worker_keeps_refusing(self): dead = _dead_url() self.offline_copy(dead)