From c3561e5e828a0f249d8b30d9ef68da28e2640b85 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 09:48:00 -0300 Subject: [PATCH] fix: refuse recovery store writes without the lock and prune idle tombstones The recovery store now asks the service lock again right before the rename, after the temporary file is written and synced, so a process that no longer holds the store lock leaves recovery.json and its memory as they were. For this store the lock counts as held only while it carries this process's token and its mtime is younger than the stale window minus one heartbeat (20 s), since a holder whose heartbeat fell behind may be in the middle of a takeover. heal() also runs the 30-day tombstone removal when memory holds an expired tombstone, so the removal holds on a store that gets no writes. Part of #444 --- apps/cli/lib/account/recovery-store.js | 89 ++++++++-- apps/cli/lib/account/server.js | 43 ++++- apps/cli/lib/commands/account.js | 12 +- apps/cli/test/account-recovery.test.js | 16 +- apps/cli/test/account.test.js | 75 ++++++++ apps/cli/test/recovery-store.test.js | 163 +++++++++++++++++- .../fix-recovery-store-lock-fence.md | 3 + docs/protocol.md | 2 +- .../fix-recovery-store-lock-fence.md | 46 +++++ packages/core/lib/store.js | 6 +- packages/core/test/store.test.js | 28 +++ 11 files changed, 440 insertions(+), 43 deletions(-) create mode 100644 docs/changelog/fix-recovery-store-lock-fence.md create mode 100644 docs/verification/fix-recovery-store-lock-fence.md diff --git a/apps/cli/lib/account/recovery-store.js b/apps/cli/lib/account/recovery-store.js index f351755c7..b5d5ce1ad 100644 --- a/apps/cli/lib/account/recovery-store.js +++ b/apps/cli/lib/account/recovery-store.js @@ -32,14 +32,24 @@ // // Every write runs in one in-process queue against a copy of the state and // first re-reads the file: the service lock goes stale after 30 s -// (account/server.js LOCK_STALE_MS) and writeJsonAtomic renames -// unconditionally, so another holder may have replaced the file meanwhile. -// guard() is asked before the write and again after it: a holder that lost -// the lock meanwhile does not acknowledge. Memory changes only once the -// write landed and guard() still answered true; a refused or failed request -// leaves the store dirty, so the next write or heal() merges and rewrites -// before it answers. guard() === false refuses that request only; fail() -// and a malformed file make the store unavailable for good. +// (account/server.js LOCK_STALE_MS), so another holder may have replaced the +// file meanwhile. guard() is asked three times: before the re-read, right +// before the rename (writeJsonAtomic's beforeRename, after the temp file is +// written and fsynced) and after the rename. A false answer before the +// rename leaves the file as it was, so a process whose event loop stalled +// past LOCK_STALE_MS (a long fsync, a suspended process) and lost the lock +// meanwhile never replaces what the new holder wrote and acknowledged. The +// answer after the rename decides whether the write is acknowledged. The +// service's guard (../commands/account.js) counts the lock as held only +// while it carries this process's token and its mtime is younger than the +// stale window minus one heartbeat, so a holder whose heartbeat fell behind +// stops writing before another process can read the lock as stale. Memory +// changes only once the write landed and guard() still answered true; a +// failed write, or one the check after the rename refused, leaves the store +// dirty, so the next write or heal() merges and rewrites before it answers. +// A refusal before the rename leaves file and memory as they were. guard() +// === false (or a throw from it) refuses that request only; fail() and a +// malformed file make the store unavailable for good. // // Rows are deleted, so a plain union (terms-store.js) would bring a removed // row back from a stale copy. Every row and every tombstone carries a @@ -79,8 +89,20 @@ // the member's last tombstone is pruned, Excluir conta included. Tombstones // pruned by a version 1 build left no floor. // -// Residual: a merge needs two copies. A copy put back while no holder runs -// is read as it is: it lacks every row written after it and holds again the +// Tombstones are pruned inside normalize(), which writes and merges reach; +// heal() also runs it when memory holds a tombstone past the TTL, so the +// 30-day removal holds on a store nobody writes to. +// +// Residual: a holder suspended in the few synchronous instructions between +// the check right before the rename and the rename itself while another +// process takes the lock over still lands its rename after the takeover +// (with the service's guard, a suspension of more than one heartbeat, or +// less when the holder had stalled past the whole stale window and its +// heartbeat refreshed the lock just before a taker that had already read +// it as stale replaced it). It answers 'indisponivel', since the check +// after the rename fails, and the new holder's heal() rewrites what it +// acknowledged, if it is still running. A merge needs two copies. A copy +// put back while no holder runs is read as it is: it lacks every row written after it and holds again the // rows removed after it, with nothing left to bury them. A copy from another // history put back while a holder runs is a union apart from memory's floor: // its rows removed in that history may come back, and those at or under @@ -430,14 +452,35 @@ function openRecoveryStore (dataDir, { now = Date.now, readOnly = false, guard = if (failed) throw fail('indisponivel') } + // guard() as a boolean: a guard that throws counts as a refusal at every + // call site, so a throw never skips the dirty mark after a rename. + function held () { + try { + return Boolean(guard()) + } catch { + return false + } + } + // Memory changes only after the write landed and guard() still answers // true. The file may hold `next` after a failure; the store stays dirty so // the next write or heal merges and rewrites first. An unreadable file - // marks the store unavailable. + // marks the store unavailable. guard() is asked again right before the + // rename: a holder that lost the lock while it read, merged or fsynced + // leaves the file as the new holder wrote it. That refusal is for the + // request only, like the one before the re-read: the file is untouched and + // memory unchanged, so the store is neither failed nor dirty. function persist (next) { + let refused = false + const beforeRename = () => { + if (held()) return + refused = true + throw fail('indisponivel') + } try { - writeJsonAtomic(file, toJson(next), 0o600) + writeJsonAtomic(file, toJson(next), 0o600, { beforeRename }) } catch (err) { + if (refused) throw fail('indisponivel') dirty = true try { readState(file) @@ -446,7 +489,7 @@ function openRecoveryStore (dataDir, { now = Date.now, readOnly = false, guard = } throw err } - if (!guard()) { + if (!held()) { dirty = true throw fail('indisponivel') } @@ -459,7 +502,7 @@ function openRecoveryStore (dataDir, { now = Date.now, readOnly = false, guard = function reread () { usable() if (readOnly) throw fail('indisponivel') - if (!guard()) throw fail('indisponivel') + if (!held()) throw fail('indisponivel') let disk try { disk = readState(file) @@ -567,17 +610,29 @@ function openRecoveryStore (dataDir, { now = Date.now, readOnly = false, guard = }) } + // True when memory holds a tombstone past the TTL. Pruning otherwise runs + // only inside writes and merges, so without this check an idle store + // would keep a removal mark past 30 days for as long as nobody writes. + function expiredTombstone () { + const cutoff = now() - TOMBSTONE_TTL_MS + for (const id in state.removed) { + if (Date.parse(state.removed[id].removedAt) < cutoff) return true + } + return false + } + // Writes the merge of the file and memory back when the file lacks // anything this holder knows (a stale writer's rename may have dropped an - // acknowledged row or tombstone) or a previous write failed. True when it + // acknowledged row or tombstone), a previous write failed, or a tombstone + // passed the TTL (the merge prunes it and raises the floor). True when it // wrote. Never throws, never writes a file it would not have written. function heal () { return enqueue(() => { - if (failed || readOnly || !guard()) return false + if (failed || readOnly || !held()) return false let seen, disk try { seen = fingerprint(file) - if (!dirty && seen === healed) return false + if (!dirty && seen === healed && !expiredTombstone()) return false disk = readState(file) } catch (err) { // Only a malformed file is final; a failed read is tried again at diff --git a/apps/cli/lib/account/server.js b/apps/cli/lib/account/server.js index f34d08d9b..20a042f25 100644 --- a/apps/cli/lib/account/server.js +++ b/apps/cli/lib/account/server.js @@ -398,7 +398,7 @@ function checkIssuerCert (cert, keyPair, association) { const LOCK_HEARTBEAT_MS = 10000 const LOCK_STALE_MS = 30000 -// -> { file, owned(), release() }. `heartbeatMs`, `staleMs` and `onLost` are for +// -> { file, owned(), held(), release() }. `heartbeatMs`, `staleMs` and `onLost` are for // tests and the serve path; the heartbeat timer is unref'd so it never // keeps a process alive on its own. function takeLock (dataDir, { heartbeatMs = LOCK_HEARTBEAT_MS, staleMs = LOCK_STALE_MS, onLost = () => {} } = {}) { @@ -408,7 +408,7 @@ function takeLock (dataDir, { heartbeatMs = LOCK_HEARTBEAT_MS, staleMs = LOCK_ST const token = nodeCrypto.randomBytes(16).toString('hex') const body = JSON.stringify({ pid: process.pid, at: new Date().toISOString(), token }) + '\n' for (let attempt = 0; attempt < 2; attempt++) { - if (publishLock(file, body)) return heartbeat(file, token, { heartbeatMs, onLost }) + if (publishLock(file, body)) return heartbeat(file, token, { heartbeatMs, staleMs, onLost }) if (attempt > 0) break const current = readLock(file) if (current === null) continue // gone between link and read: retry once @@ -428,9 +428,22 @@ function takeLock (dataDir, { heartbeatMs = LOCK_HEARTBEAT_MS, staleMs = LOCK_ST } // Refreshes the lock's mtime every `heartbeatMs` while it is still ours; -// -> { file, owned(), release() }. owned() reads the lock file now, for a -// check that cannot wait for the next tick (the shutdown close). -function heartbeat (file, token, { heartbeatMs, onLost }) { +// -> { file, owned(), held(), release() }. owned() reads the lock file now, +// for a check that cannot wait for the next tick (the shutdown close). +// held() is stricter: ours AND an mtime younger than `staleMs` minus one +// heartbeat. A holder whose own heartbeat fell behind may be in the middle +// of a takeover (the taker saw the lock stale and is about to unlink it), +// and the margin makes held() refuse a late heartbeat before anyone can +// call the lock stale. It does not cover a holder that stalled past the +// whole window: its next tick refreshes the mtime while its token is still +// in the file, possibly just before a taker that already read the lock +// stale unlinks it (recovery-store.js, Residual). A store guarded by held() refuses +// to write until the next tick refreshes the mtime while the token is still +// ours: the request is refused (the client sees an uncertain result) and +// nothing is lost from the file. A heartbeat as long as the stale window +// (tests only) leaves no margin, so the window alone applies. +function heartbeat (file, token, { heartbeatMs, staleMs, onLost }) { + const heldMs = heartbeatMs < staleMs ? staleMs - heartbeatMs : staleMs let timer = null const stop = () => { if (timer !== null) clearInterval(timer) @@ -452,6 +465,7 @@ function heartbeat (file, token, { heartbeatMs, onLost }) { return { file, owned: () => ownLock(file, token), + held: () => holdLock(file, token, heldMs), release: () => { stop() releaseLock(file, token) @@ -507,6 +521,25 @@ function ownLock (file, token) { } } +// True while the lock carries `token` and its mtime is within `windowMs` of +// now, both from one open of the file, so the token and the mtime belong to +// the same lock. +function holdLock (file, token, windowMs) { + let fd + try { + fd = fs.openSync(file, 'r') + const { mtimeMs } = fs.fstatSync(fd) + const data = JSON.parse(fs.readFileSync(fd, 'utf8')) + return isPlainObject(data) && data.token === token && lockLive({ mtimeMs }, windowMs) + } catch { + return false + } finally { + if (fd !== undefined) { + try { fs.closeSync(fd) } catch {} + } + } +} + // Removes the lock only while it carries `token`: never another's lock. function releaseLock (file, token) { if (!ownLock(file, token)) return diff --git a/apps/cli/lib/commands/account.js b/apps/cli/lib/commands/account.js index 14986c855..beabf1e67 100644 --- a/apps/cli/lib/commands/account.js +++ b/apps/cli/lib/commands/account.js @@ -894,14 +894,16 @@ async function serve (flags, env) { terms = openTermsStore(flags.dataDir, { guard: () => lock.owned(), log }) if (!terms.available()) log({ event: 'terms', reason: 'indisponivel' }) stopHeal = startTermsHeal(terms, log) - // The recovery entries (recovery.json): the same guard. An invalid file + // The recovery entries (recovery.json): every write asks lock.held(), + // ours and live by mtime, so a process whose heartbeat fell behind never + // writes while another may be taking the lock over. An invalid file // leaves only the recovery routes answering 'indisponivel'. - recovery = openRecoveryStore(flags.dataDir, { guard: () => lock.owned(), log }) + recovery = openRecoveryStore(flags.dataDir, { guard: () => lock.held(), log }) if (!recovery.available()) log({ event: 'recovery', reason: 'indisponivel' }) stopRecoveryHeal = startHeal(recovery, 'recovery', log) - // The passkey public keys and waiting deletions (passkeys.json): the same - // guard. An invalid file leaves Excluir conta and the passkey enrolment - // answering 'indisponivel'. + // The passkey public keys and waiting deletions (passkeys.json): the + // terms guard, lock.owned(). An invalid file leaves Excluir conta and + // the passkey enrolment answering 'indisponivel'. passkeys = openPasskeyStore(flags.dataDir, { guard: () => lock.owned(), log }) if (!passkeys.available()) log({ event: 'passkeys', reason: 'indisponivel' }) stopPasskeysHeal = startHeal(passkeys, 'passkeys', log) diff --git a/apps/cli/test/account-recovery.test.js b/apps/cli/test/account-recovery.test.js index 76ed43211..9e9b8d9d5 100644 --- a/apps/cli/test/account-recovery.test.js +++ b/apps/cli/test/account-recovery.test.js @@ -514,8 +514,9 @@ test('recovery: a guard lost after the write is resultado-incerto, never a defin const s = await setup(t, { guard: () => answers.length === 0 || answers.shift() }) const lookupId = hex(32) const blob = blobOf('passkey') - // reread() asks the guard before the write, persist() after it. - answers.push(true, false) + // reread() asks the guard before the write, persist() right before the + // rename and after it. + answers.push(true, true, false) const lost = await s.put(s.m, 'passkey', lookupId, blob) t.is(lost.status, 503, lost.text) t.alike(lost.body, { error: 'resultado-incerto' }) @@ -526,7 +527,7 @@ test('recovery: a guard lost after the write is resultado-incerto, never a defin t.is(retry.status, 200, retry.text) t.is((await s.fetchId(lookupId)).status, 200) - answers.push(true, false) + answers.push(true, true, false) const lostRemove = await s.remove(s.m, 'passkey', lookupId) t.is(lostRemove.status, 503, lostRemove.text) t.alike(lostRemove.body, { error: 'resultado-incerto' }) @@ -537,8 +538,9 @@ test('recovery: list and fetch after a guard lost past the rename show the file, const s = await setup(t, { guard: () => answers.length === 0 || answers.shift() }) const lookupId = hex(32) const blob = blobOf('passkey') - // reread() asks the guard before the write, persist() after the rename. - answers.push(true, false) + // reread() asks the guard before the write, persist() right before the + // rename and after it. + answers.push(true, true, false) const lost = await s.put(s.m, 'passkey', lookupId, blob) t.is(lost.status, 503, lost.text) t.alike(lost.body, { error: 'resultado-incerto' }) @@ -549,7 +551,7 @@ test('recovery: list and fetch after a guard lost past the rename show the file, t.is(listed.status, 200, listed.text) t.alike(listed.body.entries.map((e) => e.lookupId), [lookupId], 'list shows it too') - answers.push(true, false) + answers.push(true, true, false) const lostRemove = await s.remove(s.m, 'passkey', lookupId) t.is(lostRemove.status, 503, lostRemove.text) t.alike(lostRemove.body, { error: 'resultado-incerto' }) @@ -561,7 +563,7 @@ test('recovery: list and fetch after an uncertain write answer indisponivel when const answers = [] const s = await setup(t, { guard: () => answers.length === 0 || answers.shift() }) const lookupId = hex(32) - answers.push(true, false) + answers.push(true, true, false) t.is((await s.put(s.m, 'passkey', lookupId, blobOf('passkey'))).status, 503) fs.writeFileSync(path.join(s.dir, FILE), 'not json') const fetched = await s.fetchId(lookupId) diff --git a/apps/cli/test/account.test.js b/apps/cli/test/account.test.js index b0ead5726..9e15ab6bf 100644 --- a/apps/cli/test/account.test.js +++ b/apps/cli/test/account.test.js @@ -25,6 +25,7 @@ const { createGoogle } = require('../lib/account/google') const { openAccountStore, phoneSub, COOLDOWN_MS, PENDING_TTL_MS } = require('../lib/account/store') const { openTermsStore } = require('../lib/account/terms-store') const { openInvitesStore, inviteId } = require('../lib/account/invites-store') +const { openRecoveryStore, FILE: RECOVERY_FILE } = require('../lib/account/recovery-store') const { createAccountServer, keyedWindow, takeLock, lockHeld, loadIssuer, LOCK_FILE, KEY_FILE, CERT_FILE, COOKIE, MAX_SOCKETS, DEFAULT_RATE, DEFAULT_BURST } = require('../lib/account/server') const { readEnv, closeIfOwned, loseLock, startTermsHeal, startHeal, storeUnavailable } = require('../lib/commands/account') const { UsageError } = require('../lib/cli') @@ -933,6 +934,80 @@ test('data-dir lock: owned() reads the lock now, and the shutdown close runs onl t.alike(events, [{ event: 'close', reason: 'falhou' }], 'logged without detail') }) +test('data-dir lock: held() is true only while the token is ours and the mtime is live', (t) => { + const dir = path.join(tmpDir(t), 'account') + const lockFile = path.join(dir, LOCK_FILE) + // No heartbeat tick during the test: the mtime only moves when set here. + const lock = takeLock(dir, { heartbeatMs: 60 * 60 * 1000 }) + t.teardown(() => lock.release(), { force: true }) + t.is(lock.held(), true, 'ours and fresh') + + const old = (Date.now() - 31 * 1000) / 1000 + fs.utimesSync(lockFile, old, old) + t.is(lock.held(), false, 'ours but stale: another process may be taking it over') + t.is(lock.owned(), true, 'owned() still answers by token alone') + const now = new Date() + fs.utimesSync(lockFile, now, now) + t.is(lock.held(), true, 'a refreshed mtime makes it held again') + + fs.writeFileSync(lockFile, JSON.stringify({ pid: process.pid, at: new Date().toISOString(), token: 'f'.repeat(32) }) + '\n') + t.is(lock.held(), false, 'another token') + fs.unlinkSync(lockFile) + t.is(lock.held(), false, 'no lock file') +}) + +test('data-dir lock: held() keeps one heartbeat of margin before the stale window', (t) => { + const dir = path.join(tmpDir(t), 'account') + const lockFile = path.join(dir, LOCK_FILE) + // The service's windows; the test ends well before the first tick. + const lock = takeLock(dir, { heartbeatMs: 10 * 1000, staleMs: 30 * 1000 }) + t.teardown(() => lock.release(), { force: true }) + const age = (ms) => { + const at = (Date.now() - ms) / 1000 + fs.utimesSync(lockFile, at, at) + } + age(15 * 1000) + t.is(lock.held(), true, 'one heartbeat late: still held') + age(25 * 1000) + t.is(lock.held(), false, 'within one heartbeat of the stale window: not held') + t.is(lock.owned(), true, 'owned() still answers by token alone') +}) + +test('data-dir lock: the recovery store guarded by held() refuses a put while the lock is stale', async (t) => { + const dir = path.join(tmpDir(t), 'account') + const lockFile = path.join(dir, LOCK_FILE) + const lock = takeLock(dir, { heartbeatMs: 60 * 60 * 1000 }) + t.teardown(() => lock.release(), { force: true }) + const recovery = openRecoveryStore(dir, { guard: () => lock.held() }) + const member = b4a.toString(crypto.randomBytes(32), 'hex') + const entry = (kind) => ({ + member, + kind, + lookupId: b4a.toString(crypto.randomBytes(32), 'hex'), + blob: { v: 1, kind, nonce: b4a.toString(crypto.randomBytes(24), 'hex'), ciphertext: b4a.toString(crypto.randomBytes(82), 'hex') } + }) + const first = entry('passkey') + t.is((await recovery.put(first)).changed, true, 'held: the put lands') + const file = path.join(dir, RECOVERY_FILE) + const before = fs.readFileSync(file) + + const old = (Date.now() - 31 * 1000) / 1000 + fs.utimesSync(lockFile, old, old) + const second = entry('passkey') + try { + await recovery.put(second) + t.fail('a put under a stale lock resolved') + } catch (err) { + t.is(err.code, 'indisponivel', 'a put under a stale lock is refused') + } + t.ok(b4a.equals(fs.readFileSync(file), before), 'the file is unchanged') + t.ok(recovery.available(), 'the store is not failed') + + const now = new Date() + fs.utimesSync(lockFile, now, now) + t.is((await recovery.put(second)).changed, true, 'the next heartbeat makes writes land again') +}) + test('data-dir lock: a lost lock marks the store unavailable before the service stops', async (t) => { const dir = path.join(tmpDir(t), 'account') const store = openAccountStore(dir) diff --git a/apps/cli/test/recovery-store.test.js b/apps/cli/test/recovery-store.test.js index 0acccf5c0..9a423e65a 100644 --- a/apps/cli/test/recovery-store.test.js +++ b/apps/cli/test/recovery-store.test.js @@ -334,7 +334,7 @@ test('guard() false after the write does not acknowledge and heal rewrites', asy const { store, file, open } = setup(t, { guard: () => ++calls !== refuseAt }) const m = hex32() const r = req(m) - refuseAt = 2 // before = 1, after = 2 + refuseAt = 3 // before = 1, right before the rename = 2, after = 3 await rejects(t, store.put(r), 'indisponivel', 'refused after the write') t.ok(store.available()) t.ok(fs.existsSync(file), 'the file may hold the unacknowledged write') @@ -408,13 +408,13 @@ test('reads after a write refused past the rename show the file, never stale mem const kept = req(m) await store.put(kept) const r = req(m) - refuseAt = calls + 2 // before = +1, after the rename = +2 + refuseAt = calls + 3 // before = +1, right before the rename = +2, after it = +3 await rejects(t, store.put(r), 'indisponivel', 'put refused after the rename') t.ok(readFile(file).entries[r.lookupId], 'the file holds the row') t.alike(store.fetch(r.lookupId), { member: m, kind: 'passkey', blob: r.blob }, 'fetch shows the file') t.alike(store.list(m).map((x) => x.lookupId).sort(), [kept.lookupId, r.lookupId].sort(), 'list shows the file') - refuseAt = calls + 2 + refuseAt = calls + 3 await rejects(t, store.remove({ member: m, lookupId: kept.lookupId }), 'indisponivel', 'remove refused after the rename') t.ok(readFile(file).removed[kept.lookupId], 'the file holds the tombstone') t.is(store.fetch(kept.lookupId), null, 'fetch shows the removal') @@ -482,7 +482,7 @@ test('a read after an uncertain write refuses when the file cannot be read', asy const { store, file } = setup(t, { guard: () => ++calls !== refuseAt }) const m = hex32() const r = req(m) - refuseAt = 2 + refuseAt = 3 await rejects(t, store.put(r), 'indisponivel', 'put refused after the rename') fs.writeFileSync(file, 'not json') t.exception(() => store.fetch(r.lookupId), /indisponivel/) @@ -625,11 +625,12 @@ test('code rows tied on gen after a stale rename both stay: the acknowledged cod const ack = req(m, 'code', ackId) t.is((await fresh.put(ack)).changed, true) // The holder that lost its lock had read the file before that write; its - // rename lands afterwards with its own code at the same gen. + // rename lands afterwards with its own code at the same gen (it passed the + // check right before the rename and was suspended before the rename). const afterAck = fs.readFileSync(file, 'utf8') fs.writeFileSync(file, before) let calls = 0 - const stale = open({ guard: () => ++calls < 2 }) + const stale = open({ guard: () => ++calls < 3 }) await rejects(t, stale.put(req(m, 'code', staleId)), 'indisponivel', 'the stale holder does not acknowledge') const disk = readFile(file) t.is(disk.entries[staleId].gen, JSON.parse(afterAck).entries[ackId].gen, 'the two writes tie on gen') @@ -758,7 +759,7 @@ test('a read after an uncertain write never brings back a row under the floor', c.t += TOMBSTONE_TTL_MS + DAY await store.put(req(m)) t.ok(readFile(file).floors[m], 'the floor is on disk') - refuseAt = calls + 2 // before = +1, after the rename = +2 + refuseAt = calls + 3 // before = +1, right before the rename = +2, after it = +3 await rejects(t, store.put(req(m)), 'indisponivel', 'put refused after the rename') fs.writeFileSync(file, old) t.is(store.fetch(r.lookupId), null, 'fetch merges the old copy and keeps the row gone') @@ -900,3 +901,151 @@ test('the floor drops only rows at or under it, of its member, that its own copy t.alike(Object.keys(disk.entries).sort(), [own, high, otherMember].sort()) t.alike(disk.floors, { [m]: 5 }) }) + +test('heal() keeps the 30-day removal on an idle store: the tombstone goes and the floor rises', async (t) => { + const { store, file, c, open } = setup(t) + const m = hex32() + const r = req(m) + await store.put(r) + const withRow = fs.readFileSync(file, 'utf8') + c.t += 1000 + await store.remove({ member: m, lookupId: r.lookupId }) + t.is(await store.heal(), false, 'the file equals memory') + const tombGen = readFile(file).removed[r.lookupId].gen + // Only heal() runs from here on: nobody writes. + c.t += TOMBSTONE_TTL_MS + DAY + t.is(await store.heal(), true, 'heal prunes and writes') + const disk = readFile(file) + t.absent(disk.removed[r.lookupId], 'the tombstone is gone from the file') + t.is(disk.floors[m], tombGen, 'the member\'s floor is raised to its gen') + t.is(await store.heal(), false, 'nothing more to heal') + + const reopened = open() + t.is(reopened.fetch(r.lookupId), null, 'a reopened store does not bring the row back') + t.alike(readFile(file).removed, {}, 'and holds no tombstone') + fs.writeFileSync(file, withRow) + c.t += 1000 + t.is(await reopened.heal(), true, 'an older copy holding the row is rewritten') + t.is(reopened.fetch(r.lookupId), null, 'the floor keeps the row gone') + t.absent(readFile(file).entries[r.lookupId]) +}) + +test('heal() on a clean store with no expired tombstone writes nothing', async (t) => { + const { store, file, c } = setup(t) + const m = hex32() + const kept = req(m) + const gone = req(m) + await store.put(kept) + await store.put(gone) + c.t += 1000 + await store.remove({ member: m, lookupId: gone.lookupId }) + t.is(await store.heal(), false) + const bytes = fs.readFileSync(file) + const { mtimeMs, ino } = fs.statSync(file) + c.t += TOMBSTONE_TTL_MS - DAY + t.is(await store.heal(), false, 'a tombstone younger than the TTL stays') + t.ok(b4a.equals(fs.readFileSync(file), bytes), 'the file is byte-identical') + const st = fs.statSync(file) + t.is(st.mtimeMs, mtimeMs, 'and was not rewritten') + t.is(st.ino, ino) + t.ok(readFile(file).removed[gone.lookupId], 'the tombstone is still there') +}) + +test('a writer that lost the lock before the rename leaves the new holder\'s acknowledged write in place', async (t) => { + for (const op of ['put', 'remove']) { + const { file, c, dir } = setup(t) + let held = true + let inject = null + const now = () => { + if (inject) { + const f = inject + inject = null + f() + } + return c.t + } + const a = openRecoveryStore(dir, { now, guard: () => held }) + const m = hex32() + const first = req(m) + await a.put(first) + // The new holder B starts from the same file (a copy of the directory) + // and acknowledges its own write there. + const copy = path.join(tmpDir(t), 'account') + fs.mkdirSync(copy, { mode: 0o700 }) + fs.copyFileSync(file, path.join(copy, FILE)) + const b = openRecoveryStore(copy, { now: c.now }) + const theirs = req(m) + c.t += 1000 + if (op === 'put') t.is((await b.put(theirs)).changed, true, op + ': B acknowledges') + else t.alike(await b.remove({ member: m, lookupId: first.lookupId }), { changed: true }, op + ': B acknowledges') + const bFile = fs.readFileSync(path.join(copy, FILE)) + // A stalls after its first guard check and read: meanwhile B takes the + // lock and its file lands in place. + inject = () => { + fs.copyFileSync(path.join(copy, FILE), file) + held = false + } + const mine = req(m) + await rejects(t, a.put(mine), 'indisponivel', op + ': A does not acknowledge') + t.ok(b4a.equals(fs.readFileSync(file), bFile), op + ': the file is B\'s') + const reopened = openRecoveryStore(dir, { now: c.now }) + if (op === 'put') t.ok(reopened.fetch(theirs.lookupId), op + ': B\'s row is there after a reopen') + else t.is(reopened.fetch(first.lookupId), null, op + ': B\'s removal is there after a reopen') + t.is(reopened.fetch(mine.lookupId), null, op + ': A\'s refused row is not') + t.ok(a.available(), op + ': A is not failed') + } +}) + +test('a replaced code refused before the rename leaves the acknowledged code live', async (t) => { + const { file, c, dir, open } = setup(t) + let held = true + let inject = null + const now = () => { + if (inject) { + const f = inject + inject = null + f() + } + return c.t + } + const store = openRecoveryStore(dir, { now, guard: () => held }) + const m = hex32() + const acked = req(m, 'code') + await store.put(acked) + const before = fs.readFileSync(file) + // The guard answers false from the merge inside reread() on. + inject = () => { held = false } + c.t += 1000 + const replacement = req(m, 'code') + await rejects(t, store.put(replacement), 'indisponivel', 'the replacement is not acknowledged') + t.ok(b4a.equals(fs.readFileSync(file), before), 'the file is byte-identical') + t.alike(fs.readdirSync(dir), [FILE], 'no temp file is left') + t.ok(store.available(), 'the store is not failed') + t.is(store.fetch(replacement.lookupId), null, 'the new code does not fetch') + t.ok(store.fetch(acked.lookupId), 'the acknowledged code still does') + const reopened = open() + t.ok(reopened.fetch(acked.lookupId), 'the acknowledged code is live after a reopen') + t.is(reopened.fetch(replacement.lookupId), null, 'the new code does not fetch after a reopen') + held = true + c.t += 1000 + t.is((await store.put(replacement)).changed, true, 'a later put with the lock held works') + t.is(open().fetch(acked.lookupId), null, 'and then replaces the code') +}) + +test('a guard that throws right before the rename refuses the request only', async (t) => { + let calls = 0 + const { store, file, dir } = setup(t, { + guard: () => { + if (++calls === 2) throw new Error('lock read failed') + return true + } + }) + const m = hex32() + const r = req(m) + await rejects(t, store.put(r), 'indisponivel', 'refused') + t.absent(fs.existsSync(file), 'nothing was renamed into place') + t.alike(fs.readdirSync(dir), [], 'no temp file is left') + t.ok(store.available()) + t.is((await store.put(r)).changed, true, 'the next put lands') + t.ok(readFile(file).entries[r.lookupId]) +}) diff --git a/docs/changelog/fix-recovery-store-lock-fence.md b/docs/changelog/fix-recovery-store-lock-fence.md new file mode 100644 index 000000000..43a4e68b3 --- /dev/null +++ b/docs/changelog/fix-recovery-store-lock-fence.md @@ -0,0 +1,3 @@ +### Corrigido + +- Recuperação de conta, parte da #444 (PR `fix/recovery-store-lock-fence`): o serviço de conta apaga as marcas de remoção do `recovery.json` depois de 30 dias mesmo sem novas gravações, e um processo que perdeu a trava do diretório não grava mais o arquivo. Verificação em `docs/verification/fix-recovery-store-lock-fence.md`. diff --git a/docs/protocol.md b/docs/protocol.md index d5d15e1bb..a2a13080d 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -107,7 +107,7 @@ Member recovery: extra passkeys and the recovery code (`packages/web-member/lib/ - The account service's directory (`apps/cli/lib/account/store.js`, data dir mode `0o700`) is three kinds of file. `directory.json` is the snapshot `{ v, seq, journal, links, bySub, byUsername, usedNames }` (a file without `seq` is seq 0; `journal` only in `v` 2; `deleted`, a map from a deleted member key to `{ at }`, only once an account was deleted, see "Account deletion"), written through `writeJsonAtomic` with mode `0o600`; every link record must be whole (`sub`, `username` and `email` strings, `usernameChangedAt` and `linkedAt` parseable dates, `cert` and `usernameRecord` objects) or the open fails closed. `directory.journal.` is the journal (never named `*.log`, so log rotation does not match it; `` a random 16-byte hex id): NDJSON whose first line is the header `{ "journal": "" }` (written and fsynced before any change line) and whose later lines are one change each `{ seq, op: 'link' | 'email' | 'rename' | 'delete', member, ... }` (`delete` carries only `at`: the link leaves `links` and `bySub`, the key enters `deleted`, and every name it held stays in `usedNames` held by the deleted key, which a build without `deleted` refuses, so it is a one-way migration like the first page; the hwm counts links plus deleted rows, so a deletion never reads as a restored copy); it is created `O_EXCL|O_NOFOLLOW` and every read-write open calls `fchmod(0o600)` on it. `directory.hwm` is the high-water mark `{ seq, journal, links, names }` (`writeJsonAtomic`, mode `0o600`): the highest committed `seq` with the link and `usedNames` counts at that mark, written under the writer guard after every compaction and at close, and never lowered (a stored mark above the new one makes the store unavailable instead). Each change is one `write` and one `fsync`, and the fsynced append is the commit point; a failed journal `fsync` (or a write that may have put bytes in the file) makes the store unavailable (`indisponivel`) and the line is never adopted. `v` is `2` while the journal may hold lines and `1` after a clean close (`close()`: compaction, the hwm, a complete `v` 1 snapshot without `journal`, then the journal unlinked; a close that fails leaves the store open for a retry and the service logs `close falhou` and exits 1); a build that knows only `v` 1 refuses `v` 2 (`diretório de contas inválido`). A `v` 2 snapshot is bound to its journal: a missing or empty journal, one without a header, or one whose header carries another id fails closed at every open (read-write and read-only), so a deleted, rotated or truncated journal never makes names or Google accounts reusable. A journal file the snapshot does not name is an orphan (a crash between two steps below); readers ignore it, a read-write open removes it after its own snapshot write, only once its mtime is older than `ORPHAN_GRACE_MS` = 40000 ms (the lock's staleness plus a heartbeat) and after reading `directory.json` again (an id it names is skipped) under the writer guard; an orphan with a line past the snapshot's `seq` fails closed unless the named journal holds the identical line at that `seq` (a crash after the fold and before the unlink of a retired journal). A journal file is never truncated or rewritten once another process may hold it: the journal's start and every compaction (after `COMPACT_EVERY` = 256 lines, past `COMPACT_BYTES` = 1048576 bytes, and at every read-write open, which always publishes a fresh journal and a new snapshot naming it, also over a header-only journal) create a NEW file with a new id (header fsynced, data dir fsynced), rewrite the snapshot as `v` 2 naming it at the current `seq`, and only then unlink the previous file; lines a stalled writer landed in the previous file meanwhile are folded into the new one before and right after the unlink (through a read fd kept open across it). Before every append and every snapshot write, and again after each append's `fsync` before the line is acknowledged, the writer checks (`fstat` of its fd, `lstat` of the paths) that the journal is the same file (device, inode, link count above 0) with exactly the size it expects and that `directory.json` is the file (device, inode) it last read or wrote; any difference makes the store unavailable without writing (or without acknowledging the line just written), so another writer's open fences a stalled one and the unlink of its journal (nlink 0) ends it; the `account` service also marks the store unavailable as soon as its lock is lost. The guard and the write stay separate syscalls (no OS file locks): a write landing in the retired file after the new writer's last fold is refused to its client, never acknowledged. A snapshot rename whose data dir `fsync` failed is fsynced again before the next change is acknowledged. Readers resolve the journal by the id the snapshot names, replay the lines with `seq` above the snapshot's, ignore a torn trailing line, and read again (up to `OPEN_RETRIES` = 3 retries) when the snapshot's `v`, `seq` or journal id changed while they read; a gap in `seq` or a line that breaks an invariant fails closed. A read-write open whose snapshot+journal end below the hwm refuses with `diretório de contas restaurado de uma cópia antiga; confira o backup` (code `RESTAURADO`): by `seq` for a snapshot with `seq`, by the link and `usedNames` counts alone for a snapshot without it (written by an older build, whose `seq` restarted; one that passes continues the numbering from the mark's `seq`); a read-only open reports it (`restored`) and still reads. Read-only opens never write, unlink, chmod or create anything. Never rotate or delete a `directory.journal.*`; back up the three files together with the service stopped, but restore only `directory.json` and its `directory.journal.*` from the latest backup and KEEP the volume's existing `directory.hwm` (restore the backup's hwm only if the volume lost it), since overwriting the live mark with the backup's would lower it and the restored-copy refusal could never fire (an operator who deliberately restores an older copy deletes `directory.hwm`, README). A phone link is a link record with `sub` `'telefone:'` plus the member key (lowercase hex64, `phoneSub` in `apps/cli/lib/account/store.js`) and `email` `''`, written only by a phone finish (`store.finish` with `phone: true`). The label passes `SUB_RE` (`packages/core/lib/account-link.js`), `validLink` and `indexes()`, so it needs no new journal op, field or migration: an older build reads the record as an ordinary link. `bySub` stays one sub to one member, including an earlier member's Google sub. Nothing erases that sub or its e-mail. The number is not stored in the directory. `POST /account/finish` (`apps/cli/lib/account/server.js`) is in phone mode when phone verification is on and the pending row is still phase `'reserve'` with no `sub`, on the cookie path and on the app `V` path (`x-varanda-link`). `V` (`x-varanda-link`) is the bearer proof of an app row. The member Google sign-in is retired, so the redeem code `R` (`x-varanda-redeem`) is never minted. `POST /account/authorize` answers 410 `versao-antiga` before any configuration, proof, store check or outbound request, and builds no Google URL: an installed old build reads that 4xx as a definite refusal and does not retry. `GET /account/callback` reads no query, makes no code exchange and no outbound request, logs `{ event: 'callback', reason: 'aposentado' }` and redirects to `/?conta=erro`. `POST /account/authorize`, `GET /account/callback`, `GET /account/pending` and `GET /account/app/conta` answer the same way when the issuer is missing: they never call `configured()`. The `google` option of `createAccountServer` is ignored. The moderator panel (`/account/mod/*`) keeps its own Google login (`googleMod`, redirect `origin + '/account/mod/callback'`). `VARANDA_GOOGLE_CLIENT_ID` and `VARANDA_GOOGLE_CLIENT_SECRET` stay required because only that panel uses them. With phone verification off the service logs one line, event `cadastro`, which the human line prints as `cadastro de novas contas desligado: verificação de celular não configurada`. The finish statement is verified with `sub` = `phoneSub(member)`. Order after that proof: the finish limit, the age check, the phone check (403 `telefone-pendente` until the member holds a live sealed row), the member added to the server-local linking set (`linking`, a map from member to a count of phone finishes in flight), `await phoneRoutes.settled(member)` (every phone-store call queued before that wait commits or fails; `phones.settled`), the phone check again, then `store.finish({ signature, phone: true })`. The count is dropped in a `finally` (a count above 1 is only decremented). The linked check the phone routes read, on each request, is `store.status(m).linked || linking.has(m)`. A record of the same number for another member queued before this member entered `linking` can still end the row: `settled` lets it land and the second phone check answers `telefone-pendente` with no link. A record queued after the member is in `linking` sees the member as linked and answers `telefone-em-uso`. While the member is in `linking`, `POST /account/start` for that member answers 400 `pedido-invalido` before it touches the store, an invite hold or an app row, and with invites on the row is marked as finishing before the wait. With invites on, `finishInvited` resolves the subject with `memberForSub(row.sub ?? phoneSub(row.member))`, because a phone row has no `sub` yet. An invite code is spent only when the link commits; an unverified member keeps the pending row and the code (403 `telefone-pendente`). `GET /account/pending` answers 404 `expirado` for every caller (a cookie, an app `V`, a redeem code or none) and reads and writes nothing. `GET /account/app/conta` answers 404 `expirado` with or without a query, never echoes the query and never logs it. A finish that is not a phone finish answers 404 `expirado` once the body has a signature and a pending row, and before that signature is verified: phone verification off, a row past phase `'reserve'`, or a row that already holds a `sub` (a subject left by the retired Google callback, which is never finished). A finish with no issuer still answers 503 `indisponivel`. `store.finish` with `phone: true` accepts only a live row in phase `'reserve'` with no `sub`. `account phone --username ` (`lookupPhone` in `apps/cli/lib/commands/account.js`) is read-only and prints at most one JSON line for the name's single holder (`nameHolder`, current or past; a leading `@` is stripped). `kind` is `membro`, `excluido` or `pagina`. A member line is `{ username, key, kind, current, deletedAt, phone }`, and `phone` is the rows `--member` would print (an empty list when there are none). A page prints `owner` and no number, and reads no phone key; a moved page also prints `movedTo`. A page with no `pages.json` row fails with `página sem registro em pages.json`. An unknown name fails with `nome não encontrado`. - `/usage.json` (`apps/cli/lib/account/usage-stats.js`) = `{ v: 1, days: [{ day, members, admitted, activeDay, activeWeek, blocks, bytes, mediaBytes, topicPosts, connections, external }] }`. `day` is the Brasília calendar day (fixed UTC−3; no DST since 2019) of the sample. The account service passes `localDay` from `packages/core/lib/activity.js` (`DAY_OFFSET_MS`, `dayStart`, `localDay`) as `dayOf`, so a history row and the seeder's `activeDay` start the day at the same instant. Each count is a non-negative safe integer or `null` (unknown). Rows are strictly increasing by `day`, one per day, and only the newest `MAX_DAYS` = 400 are kept. The file is replaced whole through `writeJsonAtomic` with mode `0o600`, and only while the account store still allows writes. A path that is not a regular file, or a file whose JSON is not exactly that shape (a day out of order included), is never replaced: it is logged once as `{ event: 'usage', reason: 'invalido' }`, `history` in the stats view is `null`, and the operator can inspect it. A failed write is logged once as `{ event: 'usage', reason: 'falhou' }` until a later write succeeds. `PEAK_FIELDS` = `activeDay`, `activeWeek`, `connections`, `external` keep the day's maximum; `members`, `admitted`, `blocks`, `bytes`, `mediaBytes` and `topicPosts` keep the latest value; a `null` sample never erases a stored count. A sample whose day is earlier than the last row (the clock stepped back) is ignored. The background sample runs `FIRST_SAMPLE_MS` = 60000 ms after start and then every `SAMPLE_MS` = 900000 ms (15 min). Deleting the file only loses the history (the next sample starts a new one). The account service logs `{ event: 'usage', reason: 'ligada' }` when `VARANDA_STATS_EMAILS` turns the feature on and `{ event: 'usage', reason: 'desligada' }` when it is off. Those lines carry no e-mail, key, username or address. - `GET /account/mod/api/stats` (`apps/cli/lib/account/mod-routes.js`) needs a panel session whose e-mail is in `VARANDA_STATS_EMAILS`, and that list must be a subset of `VARANDA_MODERATOR_EMAILS`: `deploy.sh` refuses a list that is not, and so does the account service at start (`VARANDA_STATS_EMAILS inválido`, exit 1); the route also keeps only addresses that are moderators. Anyone else with a session gets 403 `{ "error": "nao-autorizado" }`; a request with no session stays the existing 401 `{ "error": "nao-autorizado" }`. The body is exactly `{ at, members: { total, max }, seeder, signups: { weeks, months }, history }`. `at` is the view time as a canonical ISO string. `members.total` is the number of linked accounts and `members.max` is the member cap or `null`. `seeder` is `null` until a read of `GET /health` succeeds (default `http://127.0.0.1:8788`; loopback `http` with an explicit port only, and the URL is read only while stats are on); otherwise `{ at, uptimeSeconds, admitted, activeDay, activeWeek, blocks, bytes, mediaBytes, topicPosts, connections, external }`, each count a non-negative safe integer or `null` (an older seeder without `activeDay` / `activeWeek` yields `null` for those two). A panel view triggers at most one such read per `FRESH_MS` = 300000 ms (5 min), and concurrent views share the in-flight sample. `signups.weeks` is 12 `{ start, count }` buckets, the weeks ending with the one that contains today, `start` the Monday of that week as `YYYY-MM-DD`; `signups.months` is 12 `{ month, count, total }` buckets, the calendar months ending with the current one, `month` as `YYYY-MM` and `total` the cumulative sign-ups through that month. Both bucket by the Brasília calendar day (fixed UTC−3; no DST since 2019) of `linkedAt` (`localDay`). The sign-up counts are exact: there is no small-cell suppression, because `linkedAt` is the `date` on the member's public certificate. `history` is the `usage.json` rows, oldest first, or `null` when the file is invalid. The body has no report, pending or blocked counts, and never a key, an e-mail, a sub, a username or an address. `GET /account/mod/api/me` carries `stats: true` only for a stats admin; the key is omitted for everyone else, so the unauthenticated body `deploy.sh` probes (`{"error":"nao-autorizado"}\n`, HTTP 401) is unchanged. -- `/recovery.json` (`apps/cli/lib/account/recovery-store.js`) holds the member recovery entries (section 1, "Web member storage (B2)"). It is ONE sidecar file like `terms.json`, mode `0o600`, replaced whole through `writeJsonAtomic`, and never part of `directory.json`, `directory.journal.*` or `directory.hwm`, so older and newer builds keep opening the same data dir. The shape is exactly `{ v: 2, seq, entries, removed, floors }`: `seq` is a non-negative safe integer, the store-wide generation counter; `entries` maps a `lookupId` (hex64) to `{ member, kind, nonce, ciphertext, createdAt, gen }` and `removed` maps a `lookupId` to the tombstone `{ member, removedAt, gen }`; an id is never both, and a `gen` above `seq` makes the file malformed. `floors` maps a member key (hex64) to a positive safe integer never above `seq`: the highest `gen` among that member's pruned tombstones. A version 1 file (exactly `{ v: 1, seq, entries, removed }`, no `floors`) is still read as if `floors` were empty; the first write that changes it writes version 2, which an older build refuses as malformed (its recovery routes answer `indisponivel` until the newer build returns). Opening never creates, mkdirs or chmods anything (a missing file means no entry yet). Validation is total: a file the module would not have written is never rewritten and every recovery call then fails with `indisponivel` until an operator repairs it, while every other route of the service stays up. Every write first re-reads the file and asks the service lock before and after it. Generations are ONE store-wide monotonic sequence, never per id: every put, every remove and every tombstone the normalisation writes takes `gen` = ++`seq`, and a merge takes `seq` = max(memory, disk) and keeps, per id, whichever of row or tombstone has the higher `gen` (ties keep memory), so a remove beats the stale row it buried and a later re-put beats the tombstone, whatever the clocks say. Of two live `'code'` rows of one member after a merge, only the strictly higher `gen` stays and the others become tombstones under fresh gens. Rows tied at the highest `gen` (a stale writer that lost its lock and its replacement both took `seq` + 1) are all kept live: neither write is newer, and choosing by `lookupId` could delete the code that was acknowledged; the next code put buries them all. Tombstone pruning: a tombstone whose `removedAt` is older than 30 days goes; then each member keeps at most 32, dropping the lowest gens first, but the cap never drops a tombstone younger than 10 minutes (`STALE_WINDOW_MS`, which also covers a `removedAt` ahead of this clock) nor the member's newest one, so a member may briefly hold more than 32. Pruning a tombstone raises its member's floor to the tombstone's `gen`. A merge takes each member's floor as the max of both copies, and when the holder merges the file into its memory it drops a row only the file holds whose `gen` is at or under memory's floor for that member (memory removed the row and pruned its tombstone), so a stale writer's older copy never resurrects a row whose tombstone a version 2 build pruned. The file's floor never drops a row memory holds: the service takes the lock before it opens the store and fails the store for good when it loses the lock, so memory is never behind the file on their shared history, and a file whose floor passes a row in memory comes from another history (a copy put back after the service started from an older copy or from no file). Such a copy, put back while the service runs, is merged as a union apart from memory's floor. A copy put back while no holder runs is read as it is: with nothing to merge, it lacks the rows written after it and holds again the rows removed after it (README.md, "Vínculo de conta Google e @nome", describes the backup and the restore). Reads: after an uncertain write (a failed write, or the lock refused after the rename) `list` and `fetch` first re-read and merge the file, or answer `indisponivel`, never from stale memory. Caps per member: 4 live `'passkey'` rows (a 5th answers 409 `{ "error": "limite-recuperacao" }`) and 1 live `'code'` row (a new code replaces the previous one, whose id becomes a tombstone, so the old code stops opening the account on any device that has not already used it; see "Web member storage (B2)" for why it never revokes past use); tombstones never count. A `lookupId` owned by another member answers `pedido-invalido`. `account export` and `account status` do not include the file. Logs carry only `{ event: 'recovery', reason }`, never a `lookupId`, a blob, a member key or an address. +- `/recovery.json` (`apps/cli/lib/account/recovery-store.js`) holds the member recovery entries (section 1, "Web member storage (B2)"). It is ONE sidecar file like `terms.json`, mode `0o600`, replaced whole through `writeJsonAtomic`, and never part of `directory.json`, `directory.journal.*` or `directory.hwm`, so older and newer builds keep opening the same data dir. The shape is exactly `{ v: 2, seq, entries, removed, floors }`: `seq` is a non-negative safe integer, the store-wide generation counter; `entries` maps a `lookupId` (hex64) to `{ member, kind, nonce, ciphertext, createdAt, gen }` and `removed` maps a `lookupId` to the tombstone `{ member, removedAt, gen }`; an id is never both, and a `gen` above `seq` makes the file malformed. `floors` maps a member key (hex64) to a positive safe integer never above `seq`: the highest `gen` among that member's pruned tombstones. A version 1 file (exactly `{ v: 1, seq, entries, removed }`, no `floors`) is still read as if `floors` were empty; the first write that changes it writes version 2, which an older build refuses as malformed (its recovery routes answer `indisponivel` until the newer build returns). Opening never creates, mkdirs or chmods anything (a missing file means no entry yet). Validation is total: a file the module would not have written is never rewritten and every recovery call then fails with `indisponivel` until an operator repairs it, while every other route of the service stays up. Every write first re-reads the file and asks the service lock three times: before the write, right before the rename (after the temporary file is written and fsynced) and after the rename. A refusal before the rename leaves the file as it was, so a process that lost the lock while it stalled never replaces what the new holder wrote; the answer after the rename decides whether the write is acknowledged. For this file the lock counts as held only while it carries this process's token and its mtime is younger than 20 s (the 30 s stale window of the startup check minus one 10 s heartbeat), so a holder whose heartbeat fell behind stops writing before another process can read the lock as stale; a refusal before the rename leaves the file and the holder's memory as they were. What remains is a process suspended in the few synchronous instructions between the check right before the rename and the rename itself while another process takes the lock over: more than one heartbeat, or less when the holder had stalled past the 30 s window and its heartbeat refreshed the lock just before another process, which had already read it as stale, replaced it. Generations are ONE store-wide monotonic sequence, never per id: every put, every remove and every tombstone the normalisation writes takes `gen` = ++`seq`, and a merge takes `seq` = max(memory, disk) and keeps, per id, whichever of row or tombstone has the higher `gen` (ties keep memory), so a remove beats the stale row it buried and a later re-put beats the tombstone, whatever the clocks say. Of two live `'code'` rows of one member after a merge, only the strictly higher `gen` stays and the others become tombstones under fresh gens. Rows tied at the highest `gen` (a stale writer that lost its lock and its replacement both took `seq` + 1) are all kept live: neither write is newer, and choosing by `lookupId` could delete the code that was acknowledged; the next code put buries them all. Tombstone pruning: a tombstone whose `removedAt` is older than 30 days goes; then each member keeps at most 32, dropping the lowest gens first, but the cap never drops a tombstone younger than 10 minutes (`STALE_WINDOW_MS`, which also covers a `removedAt` ahead of this clock) nor the member's newest one, so a member may briefly hold more than 32. Pruning a tombstone raises its member's floor to the tombstone's `gen`. Pruning runs on every write and merge, and the periodic heal also prunes and rewrites when a tombstone passed 30 days, so the removal holds on an idle store. A merge takes each member's floor as the max of both copies, and when the holder merges the file into its memory it drops a row only the file holds whose `gen` is at or under memory's floor for that member (memory removed the row and pruned its tombstone), so a stale writer's older copy never resurrects a row whose tombstone a version 2 build pruned. The file's floor never drops a row memory holds: the service takes the lock before it opens the store and fails the store for good when it loses the lock, so memory is never behind the file on their shared history, and a file whose floor passes a row in memory comes from another history (a copy put back after the service started from an older copy or from no file). Such a copy, put back while the service runs, is merged as a union apart from memory's floor. A copy put back while no holder runs is read as it is: with nothing to merge, it lacks the rows written after it and holds again the rows removed after it (README.md, "Vínculo de conta Google e @nome", describes the backup and the restore). Reads: after an uncertain write (a failed write, or the lock refused after the rename) `list` and `fetch` first re-read and merge the file, or answer `indisponivel`, never from stale memory. Caps per member: 4 live `'passkey'` rows (a 5th answers 409 `{ "error": "limite-recuperacao" }`) and 1 live `'code'` row (a new code replaces the previous one, whose id becomes a tombstone, so the old code stops opening the account on any device that has not already used it; see "Web member storage (B2)" for why it never revokes past use); tombstones never count. A `lookupId` owned by another member answers `pedido-invalido`. `account export` and `account status` do not include the file. Logs carry only `{ event: 'recovery', reason }`, never a `lookupId`, a blob, a member key or an address. - Recovery routes (`apps/cli/lib/account/recovery-routes.js`), HTTPS from the app to the account service, never from the swarm; POST only, same origin check and 4096-byte body cap as the other account POSTs; without a usable store every one answers 503 `indisponivel`: - `POST /account/recovery/start` `{ member }` -> `{ nonce }` for a linked member (single use, 10 minutes). The nonce is stateless, like the report nonces: base64url (54 characters) of an 8-byte expiry, 16 random bytes and a 16-byte HMAC-SHA256 under a per-process secret over `'varanda-recovery-nonce-v1'` and the member, so start stores nothing and a restart expires every outstanding nonce. Only the canonical spelling is accepted. Expiry and pruning read a clock that never steps back (the highest wall-clock value seen), so a wall clock corrected backwards cannot revive a spent nonce. A step back of more than one TTL instead draws a new secret, which voids every outstanding nonce, clears the spent set and lets TTLs follow the wall clock again. There is no per-member quota on start (anyone may name any public key, so such a quota would let a stranger lock a member out of removing a leaked code); only the global bucket bounds it. - `POST /account/recovery/put` `{ member, nonce, signature, kind, lookupId, blob }`, `POST /account/recovery/remove` `{ member, nonce, signature, kind, lookupId }` and `POST /account/recovery/list` `{ member, nonce, signature }`. Bodies: put -> `{ createdAt, changed }`, remove -> `{ changed }` (`changed` false for a no-op: the entry was already there, or already gone), list -> `{ entries: [{ lookupId, kind, createdAt }] }` (never the nonce or ciphertext). For each: the nonce must be one this process minted for that member, live and not yet spent; then the statement is verified with the service's association and, for put, `blobHash` recomputed from the validated blob; a per-member window (10 per 600 s) is charged only after the proof, then the nonce is recorded as spent until its own expiry (a refused proof spends nothing, so the spent set grows only with validly signed requests), then the link is checked, then the store. A put or remove that fails after a possible write (answered 503 `resultado-incerto`: a lost lock or a failed write) is an unknown outcome: the app keeps its unknown-outcome handling and never retries by itself; listing settles it. diff --git a/docs/verification/fix-recovery-store-lock-fence.md b/docs/verification/fix-recovery-store-lock-fence.md new file mode 100644 index 000000000..d789174b6 --- /dev/null +++ b/docs/verification/fix-recovery-store-lock-fence.md @@ -0,0 +1,46 @@ +# Trava conferida antes do rename e prazo de 30 dias sem gravações no recovery.json (branch fix/recovery-store-lock-fence) + +Parte da #444, sobre a branch `fix/recovery-store-floor`. Duas mudanças no `recovery.json` (`apps/cli/lib/account/recovery-store.js`). + +## O que muda + +- **Gravação recusada a um processo sem a trava.** Antes, cada gravação perguntava a trava antes de reler o arquivo e depois do rename; o `writeJsonAtomic` fazia o rename sempre. Um processo parado mais de 30 s (`LOCK_STALE_MS`) dentro desse trecho síncrono (um fsync longo, um processo suspenso) podia perder a trava para outro processo e, ao voltar, trocar o arquivo pelo seu: a gravação que o novo dono já tinha confirmado sumia do arquivo, e o processo antigo respondia `indisponivel` embora a sua gravação estivesse lá. Agora: + - `writeJsonAtomic` (`packages/core/lib/store.js`) aceita um quarto argumento opcional, `{ beforeRename }`, chamado depois de o arquivo temporário ser escrito, sincronizado e fechado, logo antes do rename. Se ele lança, não há rename, o temporário é apagado e o destino fica como estava. Quem não passa o argumento (todos os outros usos) não muda. + - O `recovery-store.js` pergunta a trava também ali. Resposta falsa (ou erro) recusa o pedido com `indisponivel`: o arquivo fica idêntico byte a byte, a memória não muda e o arquivo de recuperação não fica indisponível de vez nem pendente, como na recusa antes da releitura. Uma trava que lança erro conta como recusa nas quatro perguntas (antes da releitura, antes do rename, depois do rename e no `heal()`). A pergunta depois do rename continua e decide se a gravação é confirmada. + - A trava ganhou `held()` (`apps/cli/lib/account/server.js`): verdadeiro só enquanto o arquivo da trava tem o token deste processo e o mtime tem menos de `LOCK_STALE_MS` menos um heartbeat (20 s no serviço), lidos numa única abertura do arquivo. Um processo cujo próprio heartbeat atrasou pode estar no meio de uma tomada da trava por outro, então não grava, e um heartbeat atrasado faz `held()` recusar antes de outro processo poder dar a trava por vencida. Com um heartbeat tão longo quanto a janela (só nos testes), vale a janela inteira. Na próxima batida, se o token ainda é dele, o mtime é renovado e as gravações voltam. Só o `recovery.json` usa `held()` (`apps/cli/lib/commands/account.js`). +- **Prazo de 30 dias num arquivo sem gravações.** As marcas de remoção só eram apagadas dentro de `normalize()`, que roda nas gravações e uniões. O `heal()` (a cada `LOCK_HEARTBEAT_MS`) saía cedo quando o arquivo não tinha mudado e não havia pendência, então num serviço sem gravações uma marca ficava além de 30 dias. Agora o `heal()` segue adiante quando a memória tem uma marca com `removedAt` mais antigo que 30 dias: a união a apaga, sobe a marca por membro (`floors`) e grava. A conferência percorre `state.removed`, sem timer novo. Fora isso o `heal()` não muda: não grava com o arquivo inválido, em modo só leitura ou sem a trava, e sem pendência e sem marca vencida responde falso sem ler o arquivo. O teto de 32 por membro só é alcançado por gravações, que já normalizam. + +## Comandos e resultados + +Da raiz, um arquivo por vez (antes, os mesmos arquivos da branch-base): + +- `node apps/cli/test/recovery-store.test.js`: 44 testes, 434 asserções, todos passam (antes 39 testes, 390 asserções). Novos: + - "heal() keeps the 30-day removal on an idle store: the tombstone goes and the floor rises" (só `heal()` depois de 31 dias: grava, a marca sai do arquivo, `floors` sobe ao `gen`, um arquivo reaberto não tem a marca nem traz a entrada de volta, nem a partir de uma cópia antiga); + - "heal() on a clean store with no expired tombstone writes nothing" (com 29 dias o arquivo fica igual byte a byte, com o mesmo mtime e inode); + - "a writer that lost the lock before the rename leaves the new holder's acknowledged write in place" (o novo dono confirma um `put` ou um `remove` numa cópia do diretório; o arquivo dele entra no lugar durante a gravação do antigo, pelo relógio injetado, e a trava do antigo vira falsa); + - "a replaced code refused before the rename leaves the acknowledged code live"; + - "a guard that throws right before the rename refuses the request only". + Cinco testes existentes contavam as chamadas da trava (antes, depois do rename) e agora contam a nova chamada do meio: "guard() false after the write does not acknowledge and heal rewrites", "reads after a write refused past the rename show the file, never stale memory", "a read after an uncertain write refuses when the file cannot be read", "code rows tied on gen after a stale rename both stay: the acknowledged code is never deleted" e "a read after an uncertain write never brings back a row under the floor". O que eles conferem não muda. +- `node apps/cli/test/account.test.js`: 63 testes, 989 asserções, todos passam (antes 60 testes, 975 asserções). Novos: "data-dir lock: held() is true only while the token is ours and the mtime is live", "data-dir lock: held() keeps one heartbeat of margin before the stale window" e "data-dir lock: the recovery store guarded by held() refuses a put while the lock is stale". +- `node apps/cli/test/account-recovery.test.js`: 16 testes, 251 asserções, passam (antes os mesmos números). As sequências de respostas da trava ganharam a chamada do meio. +- `node packages/core/test/store.test.js`: 55 testes, 368 asserções, passam (antes 54 testes, 361 asserções). Novo: "writeJsonAtomic: beforeRename runs once before the rename, and a throw keeps the target". +- `node apps/cli/test/account-delete.test.js` (15 testes, 158 asserções) e `node apps/cli/test/account-recovery-e2e.test.js` (2 testes, 100 asserções): passam. +- `npx standard` nos arquivos alterados: sem apontamentos. + +## Falha sem a correção + +Conferido em 17/11/2026, com o código restaurado e os arquivos rodados de novo, todos passando: + +- Sem a pergunta antes do rename (`writeJsonAtomic` sem `beforeRename` no `recovery-store.js`): `recovery-store.test.js` com 36/44 testes, 407/434 asserções. Falham os três testes novos de gravação ("a writer that lost the lock before the rename ...": "the file is B's" e "B's row is there after a reopen" nos dois casos; "a replaced code refused before the rename ...": "the file is byte-identical", "the acknowledged code is live after a reopen"; "a guard that throws right before the rename ...": "nothing was renamed into place") e os cinco testes existentes que contam a chamada do meio. +- Com `held()` sem a regra do mtime (só o token): `account.test.js` com 60/62 testes, 982/986 asserções; falham "ours but stale: another process may be taking it over" e, no teste do arquivo de recuperação, "a put under a stale lock resolved" e "the file is unchanged". +- Com `held()` sem a margem de um heartbeat (janela de 30 s inteira): `account.test.js` com 62/63 testes, 988/989 asserções; falha "within one heartbeat of the stale window: not held". +- Sem a condição da marca vencida no `heal()`: `recovery-store.test.js` com 43/44 testes, 430/434 asserções; falham "heal prunes and writes", "the tombstone is gone from the file", "the member's floor is raised to its gen" e "and holds no tombstone". +- Com `writeJsonAtomic` sem chamar `beforeRename`: `store.test.js` com 54/55 testes, 365/368 asserções. + +## Limites + +- Fica uma janela de poucas instruções síncronas entre a pergunta logo antes do rename e o rename: um processo suspenso exatamente ali enquanto outro pega a trava ainda faz o rename depois disso. Com a margem, a suspensão precisa passar de um heartbeat (10 s), ou menos quando o processo já tinha parado por mais que a janela de 30 s e a batida dele renovou o mtime logo antes de outro processo, que já tinha lido a trava vencida, trocá-la. Ele responde `indisponivel` (a pergunta depois do rename falha), e o `heal()` do novo dono, se ainda estiver no ar, regrava o que confirmou. +- A escolha de `lock.held()` para o `recovery.json` em `apps/cli/lib/commands/account.js` não tem teste próprio: os testes montam o arquivo com `guard: () => lock.held()`. Voltar aquela linha para `lock.owned()` não quebra nenhum teste. +- Uma marca apagada pelo prazo de 30 dias faz o `heal()` gravar, e o serviço registra `{ event: 'recovery', reason: 'reparado' }`, como depois de um conserto. +- Só servidor: nenhuma conferência em aparelho é necessária. +- Nenhum texto dos Termos muda: a Política de Privacidade já fala em 30 dias. diff --git a/packages/core/lib/store.js b/packages/core/lib/store.js index 143ebea0a..08270c2ed 100644 --- a/packages/core/lib/store.js +++ b/packages/core/lib/store.js @@ -657,7 +657,10 @@ function readJson (file, fallback) { // the parent directory is fsynced so the rename survives a crash. Two // concurrent writers (even across processes) each use their own temp file, // so both succeed and the last rename wins; a failure unlinks the temp file. -function writeJsonAtomic (file, obj, mode) { +// `beforeRename`, when given, runs after the temp file is fsynced and closed +// and right before the rename: a throw from it leaves the target untouched, +// so a caller can re-check a lock as late as the write allows. +function writeJsonAtomic (file, obj, mode, { beforeRename } = {}) { const dir = path.dirname(file) fs.mkdirSync(dir, { recursive: true }) const tmp = path.join(dir, '.' + path.basename(file) + '.' + process.pid + '.' + b4a.toString(randomBytes(6), 'hex') + '.tmp') @@ -672,6 +675,7 @@ function writeJsonAtomic (file, obj, mode) { } finally { fs.closeSync(fd) } + if (beforeRename) beforeRename() fs.renameSync(tmp, file) published = true } finally { diff --git a/packages/core/test/store.test.js b/packages/core/test/store.test.js index 2b2561946..395021fdc 100644 --- a/packages/core/test/store.test.js +++ b/packages/core/test/store.test.js @@ -346,6 +346,34 @@ test('writeJsonAtomic: concurrent writers both succeed and one wins', async (t) t.alike(fs.readdirSync(dir), ['config.json'], 'no temp files left') }) +test('writeJsonAtomic: beforeRename runs once before the rename, and a throw keeps the target', async (t) => { + const dir = tmpDir(t) + const file = path.join(dir, 'x.json') + store.writeJsonAtomic(file, { a: 1 }, 0o600) + const seen = [] + store.writeJsonAtomic(file, { a: 2 }, 0o600, { + beforeRename: () => { + seen.push(store.readJson(file, null)) + const temps = fs.readdirSync(dir).filter((n) => n !== 'x.json') + seen.push(temps.length === 1 ? JSON.parse(fs.readFileSync(path.join(dir, temps[0]), 'utf8')) : temps) + } + }) + t.alike(seen, [{ a: 1 }, { a: 2 }], 'called once, with the temp file written and the target not yet replaced') + t.alike(store.readJson(file, null), { a: 2 }) + + const before = fs.readFileSync(file) + t.exception(() => store.writeJsonAtomic(file, { a: 3 }, 0o600, { + beforeRename: () => { throw new Error('recusado') } + }), /recusado/) + t.ok(b4a.equals(fs.readFileSync(file), before), 'the target is unchanged') + t.alike(fs.readdirSync(dir), ['x.json'], 'no temp file is left') + + store.writeJsonAtomic(file, { a: 4 }, 0o600, {}) + store.writeJsonAtomic(file, { a: 5 }) + t.alike(store.readJson(file, null), { a: 5 }, 'no option behaves as before') + t.alike(fs.readdirSync(dir), ['x.json']) +}) + test('openQuarantine wipes leftovers and is distinct from the main store', async (t) => { await testnet(t) const dir = tmpDir(t)