Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 72 additions & 17 deletions apps/cli/lib/account/recovery-store.js
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,24 @@
//
// Every write runs in one in-process queue against a copy of the state and
// first re-reads the file: the service lock goes stale after 30 s
// (account/server.js LOCK_STALE_MS) and writeJsonAtomic renames
// unconditionally, so another holder may have replaced the file meanwhile.
// guard() is asked before the write and again after it: a holder that lost
// the lock meanwhile does not acknowledge. Memory changes only once the
// write landed and guard() still answered true; a refused or failed request
// leaves the store dirty, so the next write or heal() merges and rewrites
// before it answers. guard() === false refuses that request only; fail()
// and a malformed file make the store unavailable for good.
// (account/server.js LOCK_STALE_MS), so another holder may have replaced the
// file meanwhile. guard() is asked three times: before the re-read, right
// before the rename (writeJsonAtomic's beforeRename, after the temp file is
// written and fsynced) and after the rename. A false answer before the
// rename leaves the file as it was, so a process whose event loop stalled
// past LOCK_STALE_MS (a long fsync, a suspended process) and lost the lock
// meanwhile never replaces what the new holder wrote and acknowledged. The
// answer after the rename decides whether the write is acknowledged. The
// service's guard (../commands/account.js) counts the lock as held only
// while it carries this process's token and its mtime is younger than the
// stale window minus one heartbeat, so a holder whose heartbeat fell behind
// stops writing before another process can read the lock as stale. Memory
// changes only once the write landed and guard() still answered true; a
// failed write, or one the check after the rename refused, leaves the store
// dirty, so the next write or heal() merges and rewrites before it answers.
// A refusal before the rename leaves file and memory as they were. guard()
// === false (or a throw from it) refuses that request only; fail() and a
// malformed file make the store unavailable for good.
//
// Rows are deleted, so a plain union (terms-store.js) would bring a removed
// row back from a stale copy. Every row and every tombstone carries a
Expand Down Expand Up @@ -79,8 +89,20 @@
// the member's last tombstone is pruned, Excluir conta included. Tombstones
// pruned by a version 1 build left no floor.
//
// Residual: a merge needs two copies. A copy put back while no holder runs
// is read as it is: it lacks every row written after it and holds again the
// Tombstones are pruned inside normalize(), which writes and merges reach;
// heal() also runs it when memory holds a tombstone past the TTL, so the
// 30-day removal holds on a store nobody writes to.
//
// Residual: a holder suspended in the few synchronous instructions between
// the check right before the rename and the rename itself while another
// process takes the lock over still lands its rename after the takeover
// (with the service's guard, a suspension of more than one heartbeat, or
// less when the holder had stalled past the whole stale window and its
// heartbeat refreshed the lock just before a taker that had already read
// it as stale replaced it). It answers 'indisponivel', since the check
// after the rename fails, and the new holder's heal() rewrites what it
// acknowledged, if it is still running. A merge needs two copies. A copy
// put back while no holder runs is read as it is: it lacks every row written after it and holds again the
// rows removed after it, with nothing left to bury them. A copy from another
// history put back while a holder runs is a union apart from memory's floor:
// its rows removed in that history may come back, and those at or under
Expand Down Expand Up @@ -430,14 +452,35 @@ function openRecoveryStore (dataDir, { now = Date.now, readOnly = false, guard =
if (failed) throw fail('indisponivel')
}

// guard() as a boolean: a guard that throws counts as a refusal at every
// call site, so a throw never skips the dirty mark after a rename.
function held () {
try {
return Boolean(guard())
} catch {
return false
}
}

// Memory changes only after the write landed and guard() still answers
// true. The file may hold `next` after a failure; the store stays dirty so
// the next write or heal merges and rewrites first. An unreadable file
// marks the store unavailable.
// marks the store unavailable. guard() is asked again right before the
// rename: a holder that lost the lock while it read, merged or fsynced
// leaves the file as the new holder wrote it. That refusal is for the
// request only, like the one before the re-read: the file is untouched and
// memory unchanged, so the store is neither failed nor dirty.
function persist (next) {
let refused = false
const beforeRename = () => {
if (held()) return
refused = true
throw fail('indisponivel')
}
try {
writeJsonAtomic(file, toJson(next), 0o600)
writeJsonAtomic(file, toJson(next), 0o600, { beforeRename })
} catch (err) {
if (refused) throw fail('indisponivel')
dirty = true
try {
readState(file)
Expand All @@ -446,7 +489,7 @@ function openRecoveryStore (dataDir, { now = Date.now, readOnly = false, guard =
}
throw err
}
if (!guard()) {
if (!held()) {
dirty = true
throw fail('indisponivel')
}
Expand All @@ -459,7 +502,7 @@ function openRecoveryStore (dataDir, { now = Date.now, readOnly = false, guard =
function reread () {
usable()
if (readOnly) throw fail('indisponivel')
if (!guard()) throw fail('indisponivel')
if (!held()) throw fail('indisponivel')
let disk
try {
disk = readState(file)
Expand Down Expand Up @@ -567,17 +610,29 @@ function openRecoveryStore (dataDir, { now = Date.now, readOnly = false, guard =
})
}

// True when memory holds a tombstone past the TTL. Pruning otherwise runs
// only inside writes and merges, so without this check an idle store
// would keep a removal mark past 30 days for as long as nobody writes.
function expiredTombstone () {
const cutoff = now() - TOMBSTONE_TTL_MS
for (const id in state.removed) {
if (Date.parse(state.removed[id].removedAt) < cutoff) return true
}
return false
}

// Writes the merge of the file and memory back when the file lacks
// anything this holder knows (a stale writer's rename may have dropped an
// acknowledged row or tombstone) or a previous write failed. True when it
// acknowledged row or tombstone), a previous write failed, or a tombstone
// passed the TTL (the merge prunes it and raises the floor). True when it
// wrote. Never throws, never writes a file it would not have written.
function heal () {
return enqueue(() => {
if (failed || readOnly || !guard()) return false
if (failed || readOnly || !held()) return false
let seen, disk
try {
seen = fingerprint(file)
if (!dirty && seen === healed) return false
if (!dirty && seen === healed && !expiredTombstone()) return false
disk = readState(file)
} catch (err) {
// Only a malformed file is final; a failed read is tried again at
Expand Down
43 changes: 38 additions & 5 deletions apps/cli/lib/account/server.js
Original file line number Diff line number Diff line change
Expand Up @@ -398,7 +398,7 @@ function checkIssuerCert (cert, keyPair, association) {
const LOCK_HEARTBEAT_MS = 10000
const LOCK_STALE_MS = 30000

// -> { file, owned(), release() }. `heartbeatMs`, `staleMs` and `onLost` are for
// -> { file, owned(), held(), release() }. `heartbeatMs`, `staleMs` and `onLost` are for
// tests and the serve path; the heartbeat timer is unref'd so it never
// keeps a process alive on its own.
function takeLock (dataDir, { heartbeatMs = LOCK_HEARTBEAT_MS, staleMs = LOCK_STALE_MS, onLost = () => {} } = {}) {
Expand All @@ -408,7 +408,7 @@ function takeLock (dataDir, { heartbeatMs = LOCK_HEARTBEAT_MS, staleMs = LOCK_ST
const token = nodeCrypto.randomBytes(16).toString('hex')
const body = JSON.stringify({ pid: process.pid, at: new Date().toISOString(), token }) + '\n'
for (let attempt = 0; attempt < 2; attempt++) {
if (publishLock(file, body)) return heartbeat(file, token, { heartbeatMs, onLost })
if (publishLock(file, body)) return heartbeat(file, token, { heartbeatMs, staleMs, onLost })
if (attempt > 0) break
const current = readLock(file)
if (current === null) continue // gone between link and read: retry once
Expand All @@ -428,9 +428,22 @@ function takeLock (dataDir, { heartbeatMs = LOCK_HEARTBEAT_MS, staleMs = LOCK_ST
}

// Refreshes the lock's mtime every `heartbeatMs` while it is still ours;
// -> { file, owned(), release() }. owned() reads the lock file now, for a
// check that cannot wait for the next tick (the shutdown close).
function heartbeat (file, token, { heartbeatMs, onLost }) {
// -> { file, owned(), held(), release() }. owned() reads the lock file now,
// for a check that cannot wait for the next tick (the shutdown close).
// held() is stricter: ours AND an mtime younger than `staleMs` minus one
// heartbeat. A holder whose own heartbeat fell behind may be in the middle
// of a takeover (the taker saw the lock stale and is about to unlink it),
// and the margin makes held() refuse a late heartbeat before anyone can
// call the lock stale. It does not cover a holder that stalled past the
// whole window: its next tick refreshes the mtime while its token is still
// in the file, possibly just before a taker that already read the lock
// stale unlinks it (recovery-store.js, Residual). A store guarded by held() refuses
// to write until the next tick refreshes the mtime while the token is still
// ours: the request is refused (the client sees an uncertain result) and
// nothing is lost from the file. A heartbeat as long as the stale window
// (tests only) leaves no margin, so the window alone applies.
function heartbeat (file, token, { heartbeatMs, staleMs, onLost }) {
const heldMs = heartbeatMs < staleMs ? staleMs - heartbeatMs : staleMs
let timer = null
const stop = () => {
if (timer !== null) clearInterval(timer)
Expand All @@ -452,6 +465,7 @@ function heartbeat (file, token, { heartbeatMs, onLost }) {
return {
file,
owned: () => ownLock(file, token),
held: () => holdLock(file, token, heldMs),
release: () => {
stop()
releaseLock(file, token)
Expand Down Expand Up @@ -507,6 +521,25 @@ function ownLock (file, token) {
}
}

// True while the lock carries `token` and its mtime is within `windowMs` of
// now, both from one open of the file, so the token and the mtime belong to
// the same lock.
function holdLock (file, token, windowMs) {
let fd
try {
fd = fs.openSync(file, 'r')
const { mtimeMs } = fs.fstatSync(fd)
const data = JSON.parse(fs.readFileSync(fd, 'utf8'))
return isPlainObject(data) && data.token === token && lockLive({ mtimeMs }, windowMs)
} catch {
return false
} finally {
if (fd !== undefined) {
try { fs.closeSync(fd) } catch {}
}
}
}

// Removes the lock only while it carries `token`: never another's lock.
function releaseLock (file, token) {
if (!ownLock(file, token)) return
Expand Down
12 changes: 7 additions & 5 deletions apps/cli/lib/commands/account.js
Original file line number Diff line number Diff line change
Expand Up @@ -894,14 +894,16 @@ async function serve (flags, env) {
terms = openTermsStore(flags.dataDir, { guard: () => lock.owned(), log })
if (!terms.available()) log({ event: 'terms', reason: 'indisponivel' })
stopHeal = startTermsHeal(terms, log)
// The recovery entries (recovery.json): the same guard. An invalid file
// The recovery entries (recovery.json): every write asks lock.held(),
// ours and live by mtime, so a process whose heartbeat fell behind never
// writes while another may be taking the lock over. An invalid file
// leaves only the recovery routes answering 'indisponivel'.
recovery = openRecoveryStore(flags.dataDir, { guard: () => lock.owned(), log })
recovery = openRecoveryStore(flags.dataDir, { guard: () => lock.held(), log })
if (!recovery.available()) log({ event: 'recovery', reason: 'indisponivel' })
stopRecoveryHeal = startHeal(recovery, 'recovery', log)
// The passkey public keys and waiting deletions (passkeys.json): the same
// guard. An invalid file leaves Excluir conta and the passkey enrolment
// answering 'indisponivel'.
// The passkey public keys and waiting deletions (passkeys.json): the
// terms guard, lock.owned(). An invalid file leaves Excluir conta and
// the passkey enrolment answering 'indisponivel'.
passkeys = openPasskeyStore(flags.dataDir, { guard: () => lock.owned(), log })
if (!passkeys.available()) log({ event: 'passkeys', reason: 'indisponivel' })
stopPasskeysHeal = startHeal(passkeys, 'passkeys', log)
Expand Down
16 changes: 9 additions & 7 deletions apps/cli/test/account-recovery.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -514,8 +514,9 @@ test('recovery: a guard lost after the write is resultado-incerto, never a defin
const s = await setup(t, { guard: () => answers.length === 0 || answers.shift() })
const lookupId = hex(32)
const blob = blobOf('passkey')
// reread() asks the guard before the write, persist() after it.
answers.push(true, false)
// reread() asks the guard before the write, persist() right before the
// rename and after it.
answers.push(true, true, false)
const lost = await s.put(s.m, 'passkey', lookupId, blob)
t.is(lost.status, 503, lost.text)
t.alike(lost.body, { error: 'resultado-incerto' })
Expand All @@ -526,7 +527,7 @@ test('recovery: a guard lost after the write is resultado-incerto, never a defin
t.is(retry.status, 200, retry.text)
t.is((await s.fetchId(lookupId)).status, 200)

answers.push(true, false)
answers.push(true, true, false)
const lostRemove = await s.remove(s.m, 'passkey', lookupId)
t.is(lostRemove.status, 503, lostRemove.text)
t.alike(lostRemove.body, { error: 'resultado-incerto' })
Expand All @@ -537,8 +538,9 @@ test('recovery: list and fetch after a guard lost past the rename show the file,
const s = await setup(t, { guard: () => answers.length === 0 || answers.shift() })
const lookupId = hex(32)
const blob = blobOf('passkey')
// reread() asks the guard before the write, persist() after the rename.
answers.push(true, false)
// reread() asks the guard before the write, persist() right before the
// rename and after it.
answers.push(true, true, false)
const lost = await s.put(s.m, 'passkey', lookupId, blob)
t.is(lost.status, 503, lost.text)
t.alike(lost.body, { error: 'resultado-incerto' })
Expand All @@ -549,7 +551,7 @@ test('recovery: list and fetch after a guard lost past the rename show the file,
t.is(listed.status, 200, listed.text)
t.alike(listed.body.entries.map((e) => e.lookupId), [lookupId], 'list shows it too')

answers.push(true, false)
answers.push(true, true, false)
const lostRemove = await s.remove(s.m, 'passkey', lookupId)
t.is(lostRemove.status, 503, lostRemove.text)
t.alike(lostRemove.body, { error: 'resultado-incerto' })
Expand All @@ -561,7 +563,7 @@ test('recovery: list and fetch after an uncertain write answer indisponivel when
const answers = []
const s = await setup(t, { guard: () => answers.length === 0 || answers.shift() })
const lookupId = hex(32)
answers.push(true, false)
answers.push(true, true, false)
t.is((await s.put(s.m, 'passkey', lookupId, blobOf('passkey'))).status, 503)
fs.writeFileSync(path.join(s.dir, FILE), 'not json')
const fetched = await s.fetchId(lookupId)
Expand Down
75 changes: 75 additions & 0 deletions apps/cli/test/account.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ const { createGoogle } = require('../lib/account/google')
const { openAccountStore, phoneSub, COOLDOWN_MS, PENDING_TTL_MS } = require('../lib/account/store')
const { openTermsStore } = require('../lib/account/terms-store')
const { openInvitesStore, inviteId } = require('../lib/account/invites-store')
const { openRecoveryStore, FILE: RECOVERY_FILE } = require('../lib/account/recovery-store')
const { createAccountServer, keyedWindow, takeLock, lockHeld, loadIssuer, LOCK_FILE, KEY_FILE, CERT_FILE, COOKIE, MAX_SOCKETS, DEFAULT_RATE, DEFAULT_BURST } = require('../lib/account/server')
const { readEnv, closeIfOwned, loseLock, startTermsHeal, startHeal, storeUnavailable } = require('../lib/commands/account')
const { UsageError } = require('../lib/cli')
Expand Down Expand Up @@ -933,6 +934,80 @@ test('data-dir lock: owned() reads the lock now, and the shutdown close runs onl
t.alike(events, [{ event: 'close', reason: 'falhou' }], 'logged without detail')
})

test('data-dir lock: held() is true only while the token is ours and the mtime is live', (t) => {
const dir = path.join(tmpDir(t), 'account')
const lockFile = path.join(dir, LOCK_FILE)
// No heartbeat tick during the test: the mtime only moves when set here.
const lock = takeLock(dir, { heartbeatMs: 60 * 60 * 1000 })
t.teardown(() => lock.release(), { force: true })
t.is(lock.held(), true, 'ours and fresh')

const old = (Date.now() - 31 * 1000) / 1000
fs.utimesSync(lockFile, old, old)
t.is(lock.held(), false, 'ours but stale: another process may be taking it over')
t.is(lock.owned(), true, 'owned() still answers by token alone')
const now = new Date()
fs.utimesSync(lockFile, now, now)
t.is(lock.held(), true, 'a refreshed mtime makes it held again')

fs.writeFileSync(lockFile, JSON.stringify({ pid: process.pid, at: new Date().toISOString(), token: 'f'.repeat(32) }) + '\n')
t.is(lock.held(), false, 'another token')
fs.unlinkSync(lockFile)
t.is(lock.held(), false, 'no lock file')
})

test('data-dir lock: held() keeps one heartbeat of margin before the stale window', (t) => {
const dir = path.join(tmpDir(t), 'account')
const lockFile = path.join(dir, LOCK_FILE)
// The service's windows; the test ends well before the first tick.
const lock = takeLock(dir, { heartbeatMs: 10 * 1000, staleMs: 30 * 1000 })
t.teardown(() => lock.release(), { force: true })
const age = (ms) => {
const at = (Date.now() - ms) / 1000
fs.utimesSync(lockFile, at, at)
}
age(15 * 1000)
t.is(lock.held(), true, 'one heartbeat late: still held')
age(25 * 1000)
t.is(lock.held(), false, 'within one heartbeat of the stale window: not held')
t.is(lock.owned(), true, 'owned() still answers by token alone')
})

test('data-dir lock: the recovery store guarded by held() refuses a put while the lock is stale', async (t) => {
const dir = path.join(tmpDir(t), 'account')
const lockFile = path.join(dir, LOCK_FILE)
const lock = takeLock(dir, { heartbeatMs: 60 * 60 * 1000 })
t.teardown(() => lock.release(), { force: true })
const recovery = openRecoveryStore(dir, { guard: () => lock.held() })
const member = b4a.toString(crypto.randomBytes(32), 'hex')
const entry = (kind) => ({
member,
kind,
lookupId: b4a.toString(crypto.randomBytes(32), 'hex'),
blob: { v: 1, kind, nonce: b4a.toString(crypto.randomBytes(24), 'hex'), ciphertext: b4a.toString(crypto.randomBytes(82), 'hex') }
})
const first = entry('passkey')
t.is((await recovery.put(first)).changed, true, 'held: the put lands')
const file = path.join(dir, RECOVERY_FILE)
const before = fs.readFileSync(file)

const old = (Date.now() - 31 * 1000) / 1000
fs.utimesSync(lockFile, old, old)
const second = entry('passkey')
try {
await recovery.put(second)
t.fail('a put under a stale lock resolved')
} catch (err) {
t.is(err.code, 'indisponivel', 'a put under a stale lock is refused')
}
t.ok(b4a.equals(fs.readFileSync(file), before), 'the file is unchanged')
t.ok(recovery.available(), 'the store is not failed')

const now = new Date()
fs.utimesSync(lockFile, now, now)
t.is((await recovery.put(second)).changed, true, 'the next heartbeat makes writes land again')
})

test('data-dir lock: a lost lock marks the store unavailable before the service stops', async (t) => {
const dir = path.join(tmpDir(t), 'account')
const store = openAccountStore(dir)
Expand Down
Loading
Loading