From d25c4803831421655ad39be110c44bb66b49aed4 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 00:24:30 -0300 Subject: [PATCH 01/25] fix(groups): fold a cut writer held sparse like the whole copy A copy counts up to its first hole, and past it only the leaves and key blocks a cut keeps, once the hole is at or past the cut. A key block citing a cut CW is judged by the CW's signed length, so a copy trimmed at the cut reads the history the whole copy reads (#350). --- packages/core/lib/group.js | 38 ++++++++++++++--- packages/core/test/group-private.test.js | 53 ++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/packages/core/lib/group.js b/packages/core/lib/group.js index 33a9c4ac..8c29f897 100644 --- a/packages/core/lib/group.js +++ b/packages/core/lib/group.js @@ -1326,6 +1326,20 @@ function lastOn (events) { return !!events && events.length > 0 && events[events.length - 1].on } +// The blocks a writer's cut keeps from the cut on. +function keptPastCut (record) { + return record.type === 'group-leave' || record.type === 'group-key' +} + +// How many blocks from 0 on some copy of a core holds, valid or not. +function heldPrefix (copies) { + const held = new Set() + for (const x of copies) for (const seq of x.log.raw.keys()) held.add(seq) + let n = 0 + while (held.has(n)) n++ + return n +} + function pushTo (map, k, v) { const list = map.get(k) if (list) list.push(v) @@ -1748,7 +1762,7 @@ function bindCores (input, asCommunity) { if (!isHex64(key) || key === G) continue const log = readLog(w.entries, false, null, priv, asCommunity) if (log.header === null && !log.raw.has(0)) continue - pushTo(copies, key, { log, diary: w.diary }) + pushTo(copies, key, { log, diary: w.diary, signed: Number.isSafeInteger(w.signed) && w.signed > 0 ? w.signed : 0 }) } } // Every copy with a block 0 takes part in the merge, whatever its header @@ -1808,14 +1822,24 @@ function bindCores (input, asCommunity) { // reads, or seen by a reader whose hole hides that join) still cuts. const held = marks.cuts.get(h.diary) const cut = Math.min(sl.cut, held && held.has(key) ? held.get(key) : Infinity) - const end = Math.max(cut, 1) - const records = cut === Infinity ? log.records : log.records.filter((x) => x.seq < end || x.record.type === 'group-leave' || x.record.type === 'group-key') + const end = cut === Infinity ? Infinity : Math.max(cut, 1) + // Seeders hold a cut writer sparse, and readers read past a hole (#350): + // a copy counts up to its first hole, and past it only the blocks a cut + // keeps, once the hole is at or past the cut. A sparse copy then folds + // like the whole one, and a partial copy, cut or not, as the prefix it + // always was. + const prefix = heldPrefix(list) + const records = log.records.filter((x) => x.seq < Math.min(prefix, end) || (prefix >= end && keptPastCut(x.record))) // A CW's generation counts its header's rejoin only when the diary's // earliest on-join of it names the same leave (dropStaleLeaves). const c = { key, diary: h.diary, member: h.member, rejoin: h.rejoin, at: h.at, ...splitReacts(records) } if (cut !== Infinity) { c.cut = cut c.held = log.records.length ? log.records[log.records.length - 1].seq : 0 + // The length its writer signed, which a sparse copy proves without + // holding the blocks past the cut. + const signed = Math.max(0, ...list.map((x) => x.signed)) + if (signed > 0) c.signed = signed } c.frozen = sl.frozen c.released = sl.released @@ -2429,7 +2453,7 @@ function foldCore ({ G, header, creator, rootBound, rootEntry, now, C = null, bo const a = n.record.auth if (cs === null || a === null || holdsG(n)) return null const d = n.diary - // Judged against the CW's held length, not its length after a diary + // Judged against the CW's signed length, not its length after a diary // cut: a cut never makes the history a key block handed out unreadable // (sending still needs the CW in C's authorityWriters, which a cut CW is not). if (a.writer !== cwOf.get(n.writer) || cs.cwDiary.get(a.writer) !== d || !cs.lastSeq.has(a.writer) || a.seq > cs.heldSeq.get(a.writer) + 1) return null @@ -3394,8 +3418,10 @@ function foldCore ({ G, header, creator, rootBound, rootEntry, now, C = null, bo cwDiary: writerDiary, counted, lastSeq: state.lastSeq, - // The last seq held per counted CW before any diary cut. - heldSeq: new Map(logs.map((l) => [l.key, Math.max(state.lastSeq.get(l.key) || 0, l.held || 0)])), + // The last seq per counted CW before any diary cut: of a cut CW, the + // last its signed length proves, so a copy trimmed at the cut reads + // the history a whole copy reads (#350). + heldSeq: new Map(logs.map((l) => [l.key, Math.max(state.lastSeq.get(l.key) || 0, l.held || 0, (l.signed || 0) - 1)])), rootAdminC: rootAdmin, rawAdminC: rawAdmin, bansC: bans, diff --git a/packages/core/test/group-private.test.js b/packages/core/test/group-private.test.js index d18c71e3..fde354a2 100644 --- a/packages/core/test/group-private.test.js +++ b/packages/core/test/group-private.test.js @@ -570,6 +570,59 @@ test('a key block citing a cut community writer stays readable (#350)', (t) => { t.is(g.rekeyPending, true) }) +test('a key block citing a cut CW held sparse is judged by its signed length (#350)', (t) => { + const w = pworld({ cadmins: [B] }) + w.add(B, wB) + w.comm.push(cwKey(B), communityBan({ member: K(0xabc), on: true, cuts: [], at: AT(5) })) + const ban = w.push(G, groupBan({ member: Y, on: true, cuts: [{ writer: wY, seq: 1 }], at: AT(6) })) + const E2 = w.rotate(wB, [G, wA, wB, wX], { basis: [ban], auth: w.cauth(B), at: 7 }) + sealedPost(w, wX, E2, 8) + w.comm.mark(B, cwKey(B), 1) + const whole = w.input(w.reader(wX)) + const cwB = whole.community.writers.find((l) => l.key === cwKey(B)) + t.is(cwB.entries.length, 2, 'the auth cites the whole CW, past its cut') + const trimmed = (signed) => ({ ...whole, community: { ...whole.community, writers: whole.community.writers.map((l) => l === cwB ? { ...l, entries: l.entries.slice(0, 1), signed } : l) } }) + t.alike(buildGroup(trimmed(undefined)).posts, [], 'control: held alone, the trimmed CW is too short for the auth') + const g = buildGroup(trimmed(2)) + t.alike(g.posts.map((p) => p.text), ['p8'], 'the signed length keeps the history readable') + t.is(g.epoch, null, 'and still sends nothing') + t.is(normalize(g), normalize(buildGroup(whole)), 'the sparse CW folds like the whole one') +}) + +test('a cut writer held sparse folds like the whole copy; a copy with a hole folds as its prefix (#350)', (t) => { + const w = pworld() + const E2 = w.rotate(wA, [G, wA, wX, wY], { at: 6 }) + sealedPost(w, wA, E2, 7) + const cut = w.len(wA) + sealedPost(w, wA, E2, 8) + const E3 = w.rotate(wA, [G, wA, wX, wY], { at: 9 }) + sealedPost(w, wA, E3, 10) + w.push(wA, groupLeave({ cuts: [{ writer: wA, seq: w.len(wA) }], at: AT(11) })) + sealedPost(w, wX, E3, 12) + sealedPost(w, wX, E3, 13) + const signed = w.len(wA) + const marks = new Map([[A, [{ writer: wA, cut, seq: 2 }]]]) + const base = w.input(w.reader(wX), { marks }) + const kept = (e) => e.seq < cut || e.record.type === 'group-leave' || e.record.type === 'group-key' + const withCopy = (key, entries, extra = {}) => ({ ...base, writers: base.writers.map((l) => l.key === key ? { ...l, entries, ...extra } : l) }) + const wAentries = base.writers.find((l) => l.key === wA).entries + const wXentries = base.writers.find((l) => l.key === wX).entries + const whole = buildGroup(base) + t.alike(whole.posts.map((p) => p.text), ['p7', 'p12', 'p13'], 'control: the whole copy drops the posts past the cut') + t.is(whole.members.get(A), 'left', 'control: the leave past the cut counts') + const sparse = wAentries.filter(kept) + t.alike(sparse.map((e) => e.seq), [0, 1, 2, 4, 6], 'the sparse copy: below the cut, the key block and the leave past it') + t.is(normalize(buildGroup(withCopy(wA, sparse, { signed }))), normalize(whole), 'the sparse copy folds like the whole one') + t.is(normalize(buildGroup({ ...base, writers: [...base.writers, { key: wA, diary: A, entries: sparse, signed }] })), normalize(whole), 'and so does a sparse copy next to the whole one') + // A copy that stops below the cut counts only up to its hole, even with + // the leave and key block held past it. + const short = wAentries.filter((e) => e.seq < 2 || e.seq === 4 || e.seq === 6) + t.is(normalize(buildGroup(withCopy(wA, short, { signed }))), normalize(buildGroup(withCopy(wA, wAentries.slice(0, 2)))), 'a hole below the cut: the prefix') + // An uncut writer never reads past a hole. + const holed = wXentries.filter((e) => e.seq !== 1) + t.is(normalize(buildGroup(withCopy(wX, holed))), normalize(buildGroup(withCopy(wX, wXentries.slice(0, 1)))), 'an uncut writer: the prefix') +}) + test('a diary cut below its own leave keeps the diary left and the removal (#350)', (t) => { const unleave = (cut) => { const w = pworld() From 833bc13f5ba390f7c0ecc802479380e0d17301cd Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 00:35:00 -0300 Subject: [PATCH 02/25] feat(core): read a group writer past its first hole A node reads each group writer up to its first hole, as before, and past it only the leaves and key blocks it holds, so a cut writer a seeder keeps sparse folds like the whole copy (#350). --- packages/core/lib/member.js | 3 +- packages/core/lib/store.js | 22 ++++++++++++ packages/core/test/group-member-6.test.js | 42 +++++++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/packages/core/lib/member.js b/packages/core/lib/member.js index 96a160ef..9a5873fd 100644 --- a/packages/core/lib/member.js +++ b/packages/core/lib/member.js @@ -59,6 +59,7 @@ const { readAll, readRaw, readBlocks, + readHeldBlocks, appendRecord, appendRecords, readConfig, @@ -1352,7 +1353,7 @@ async function readCoreParts (identity, key, { blocked, revokedIssuers }, kind) const h = decodeBlock(await core.get(0, { wait: false, raw: true })) if (validateRecord(h, 0) === null) localDiaries.add(hex(core.key)) if (groups.validateWriterHeader(h) !== null || h.group !== key) continue - logs.push({ key: hex(core.key), diary: h.diary, entries: await readBlocks(core), length: core.contiguousLength, signed: core.length }) + logs.push({ key: hex(core.key), diary: h.diary, entries: await readHeldBlocks(core), length: core.contiguousLength, signed: core.length }) } finally { await core.close() } diff --git a/packages/core/lib/store.js b/packages/core/lib/store.js index 855fbeaf..f3f11107 100644 --- a/packages/core/lib/store.js +++ b/packages/core/lib/store.js @@ -520,6 +520,27 @@ async function readBlocks (core) { return out } +// A group writer's blocks as readBlocks, then past the first hole only the +// leaves and key blocks held: a seeder clears the rest of a cut writer +// (#350), and the fold reads past a hole only what a cut keeps. +async function readHeldBlocks (core) { + assertOpen(core) + const limit = Math.min(core.length, MAX_LOG_BLOCKS) + const out = [] + let hole = false + for (let i = 0; i < limit; i++) { + const block = (await core.has(i)) ? await core.get(i, { wait: false, raw: true }) : null + if (block === null) { + hole = true + continue + } + const record = decodeBlock(block) + if (hole && (record === null || (record.type !== 'group-leave' && record.type !== 'group-key'))) continue + out.push({ seq: i, record }) + } + return out +} + function assertOpen (core) { if (core.closing) { const err = new Error('SESSION_CLOSED: cannot read from a closed session') @@ -935,6 +956,7 @@ module.exports = { readAll, readRaw, readBlocks, + readHeldBlocks, appendRecord, appendRecords, readConfig, diff --git a/packages/core/test/group-member-6.test.js b/packages/core/test/group-member-6.test.js index 5a03fa7c..20a9ec5a 100644 --- a/packages/core/test/group-member-6.test.js +++ b/packages/core/test/group-member-6.test.js @@ -9,6 +9,7 @@ const { readBoundRoots } = require('../lib/sync') const store = require('../lib/store') const member = require('../lib/member') const { hex, person, exchange, pair, writerEntries, offlineAppend, rejects, FILL, createGroup, joinGroup, THUMBS, HEART } = require('./helpers/group-member') +const { groupCutRecord } = require('../lib/records') test('loading a group records a root its creator bound, and not one it never bound', async (t) => { const { b, group } = await pair(t) @@ -217,3 +218,44 @@ test('group.read leaves out the fold\'s forum keys: no topic, poll voters or thr const hidden = await member.groupRead(a, { group }) t.alike(keys(hidden), [['Sim!', false, false, false], ['', false, false, false]], 'a hidden root shows no poll') }) + +// A seeder keeps a cut writer sparse (#350): below the cut, then only its +// leaves and key blocks. A node holding such a copy folds it as the whole one. +test('a cut writer held sparse folds like the whole copy (#350)', async (t) => { + const { a, b, group } = await pair(t) + await member.groupPost(b, { group, text: 'antes' }) + const own = await writerEntries(b, group) + const cut = own.length + await offlineAppend(b, group, groupLib.groupPost({ text: 'depois' })) + await offlineAppend(b, group, groupLib.groupPost({ text: 'depois 2' })) + const leave = await offlineAppend(b, group, groupLib.groupLeave({ seen: null, cuts: [{ writer: own.key, seq: cut + 2 }] })) + const s = await store.openStore(b.dataDir) + try { + await store.appendRecord(await store.openDiary(s), groupCutRecord({ group, writer: own.key, cut })) + } finally { + await s.close() + } + await exchange(b, a) + const view = async () => { + const info = await member.groupInfo(a, { group }) + const page = await member.groupRead(a, { group }) + const members = await member.groupMembers(a, { group }) + return JSON.stringify({ info, members, posts: page.items.map((p) => [p.writer, p.seq, p.text]) }) + } + const whole = await view() + t.ok(JSON.parse(whole).posts.some((p) => p[2] === 'antes'), 'control: the post below the cut counts') + t.absent(JSON.parse(whole).posts.some((p) => p[2] === 'depois'), 'control: the posts past the cut do not') + t.absent(JSON.parse(whole).members.members.some((m) => m.member === b.key), 'control: the leave past the cut counts') + const sa = await store.openStore(a.dataDir) + try { + const core = sa.get({ key: Buffer.from(own.key, 'hex'), writable: false }) + await core.ready() + await core.clear(cut, leave.seq) + t.is(core.contiguousLength, cut, 'a sparse copy: a hole at the cut') + t.ok(await core.has(leave.seq), 'the leave past it is held') + await core.close() + } finally { + await sa.close() + } + t.is(await view(), whole, 'the sparse copy folds like the whole one') +}) From b457aa132aef24adedcc2dddf79b6f88b69815f1 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 00:40:23 -0300 Subject: [PATCH 03/25] feat(web-member): read a group writer past its first hole The browser node reads writers as the core node does: past the first hole, only the leaves and key blocks held. The fold memo stamps the blocks held past the contiguous length, and the community fold takes the diaries' cut marks, so a cut CW held sparse folds like the whole one (#350). --- packages/web-member/lib/member-ops.js | 30 +++++++-- packages/web-member/lib/node-store.js | 22 +++++++ packages/web-member/test/group-memo.test.js | 73 ++++++++++++++++++++- packages/web-member/test/group-ops.test.js | 9 ++- 4 files changed, 124 insertions(+), 10 deletions(-) diff --git a/packages/web-member/lib/member-ops.js b/packages/web-member/lib/member-ops.js index b36e4311..d8ae8370 100644 --- a/packages/web-member/lib/member-ops.js +++ b/packages/web-member/lib/member-ops.js @@ -2482,6 +2482,16 @@ async function groupCoreState (core) { return { entries: (await store.readBlocks(core)).slice(0, length), length, signed, fork } } +// A writer as groupCoreState, and past its first hole the leaves and key +// blocks held below the signed length reported (store.readHeldBlocks): a +// seeder keeps a cut writer sparse (#350). +async function writerCoreState (core) { + const length = core.contiguousLength + const signed = core.length + const fork = core.fork + return { entries: (await store.readHeldBlocks(core)).filter((e) => e.seq < signed), length, signed, fork } +} + // A stored group core's entries (store.readBlocks: no diary deletion fold), // read-only -> { entries, length, fork } or null. async function readGroupCore (ctx, key, rec = null) { @@ -2528,8 +2538,10 @@ function memoChecks (ctx) { return ctx.memoCheck === true || MEMO_ENV_CHECK } +// A writer is read past its first hole too (groupCoreState), so a group +// core's stamp is a replica's. function groupStamp (core) { - return core.contiguousLength + ',' + core.length + ',' + core.fork + return replicaStamp(core) } // readVisible also shows the stored blocks past a hole, so a replica's @@ -2556,8 +2568,8 @@ async function withNotedGroupCore (ctx, key, rec, read) { if (rec === null) return withGroupCore(ctx, key, (core) => read(core, noop)) let stamp = 'none' let skip = false - const value = await withGroupCore(ctx, key, (core) => { - stamp = groupStamp(core) + const value = await withGroupCore(ctx, key, async (core) => { + stamp = await groupStamp(core) return read(core, () => { skip = true }) }) if (!skip) noteDep(rec, 'k', key, stamp) @@ -2751,7 +2763,7 @@ async function readWriterCoreOf (ctx, key, group, rec = null) { if (block !== null) never() return null } - return groupCoreState(core) + return writerCoreState(core) }) } @@ -2900,7 +2912,7 @@ async function foldGroup (ctx, base, group, asPage, kv, rec) { if (core === null) continue const h = core.entries[0].record if (validateWriterHeader(h) !== null || h.group !== group) continue - writers.push({ key, entries: core.entries, diary: h.diary }) + writers.push({ key, entries: core.entries, diary: h.diary, signed: core.signed }) cores.set(key, { diary: h.diary, at: h.at, length: core.length, signed: core.signed, entries: core.entries }) } const diaries = new Map() @@ -4864,21 +4876,25 @@ async function foldCommunity (ctx, base, key, kv, rec) { const core = await readWriterCoreOf(ctx, k, key, rec) if (core === null) continue const h = core.entries[0].record - writers.push({ key: k, entries: core.entries, diary: h.diary }) + writers.push({ key: k, entries: core.entries, diary: h.diary, signed: core.signed }) cores.set(k, { diary: h.diary, at: h.at, length: core.length, signed: core.signed, entries: core.entries }) } const diaries = new Map() const joins = new Map() + // The diaries' group-cut marks for C, as the core node folds it: a cut CW + // held sparse folds like the whole one only with its cut (#350). + const marks = new Map() const confirms = new Map() for (const d of new Set([ownHex, header.diary, ...writers.map((w) => w.diary)])) { const entries = await foldEntriesOf(ctx, base, d, rec) diaries.set(d, entries) if (entries !== null) { joins.set(d, groupJoinHistory(entries, key)) + marks.set(d, groupCutHistory(entries, key)) confirms.set(d, groupConfirmHistory(entries, key)) } } - const input = { root: { key, entries: root.entries }, writers, joins, confirms } + const input = { root: { key, entries: root.entries }, writers, joins, marks, confirms } const built = grp.buildCommunity({ ...input, now: Date.now() }) return { key, group: key, header, own, built, cores, diaries, joins, input } } diff --git a/packages/web-member/lib/node-store.js b/packages/web-member/lib/node-store.js index 4b8e92a0..119c6582 100644 --- a/packages/web-member/lib/node-store.js +++ b/packages/web-member/lib/node-store.js @@ -611,6 +611,27 @@ async function readBlocks (core) { return out } +// A group writer's blocks as readBlocks, then past the first hole only the +// leaves and key blocks held: a seeder clears the rest of a cut writer +// (#350), and the fold reads past a hole only what a cut keeps. +async function readHeldBlocks (core) { + assertOpen(core) + const limit = Math.min(core.length, MAX_LOG_BLOCKS) + const out = [] + let hole = false + for (let i = 0; i < limit; i++) { + const block = (await core.has(i)) ? await core.get(i, { wait: false, raw: true }) : null + if (block === null) { + hole = true + continue + } + const record = decodeBlock(block) + if (hole && (record === null || (record.type !== 'group-leave' && record.type !== 'group-key'))) continue + out.push({ seq: i, record }) + } + return out +} + function assertOpen (core) { if (core.closing) { const err = new Error('SESSION_CLOSED: cannot read from a closed session') @@ -2354,6 +2375,7 @@ module.exports = { readRaw, readHeld, readBlocks, + readHeldBlocks, readBlock0, appendRecord, appendRecords, diff --git a/packages/web-member/test/group-memo.test.js b/packages/web-member/test/group-memo.test.js index 678df38e..1945c01f 100644 --- a/packages/web-member/test/group-memo.test.js +++ b/packages/web-member/test/group-memo.test.js @@ -15,13 +15,15 @@ const { createGroupKeys } = require('../lib/group-keys') const { createBrowserSession } = require('../lib/session') const ops = require('../lib/member-ops') const store = require('../lib/node-store') +const { groupCutRecord } = require('@varanda/core/lib/records') +const { communityBan } = require('@varanda/core/lib/group') const hex = (buf) => b4a.toString(buf, 'hex') -// Block decodes of group cores (readBlocks) and diary replicas +// Block decodes of group cores (readBlocks, readHeldBlocks) and diary replicas // (readVisible), the work the memo saves. const reads = { n: 0 } -for (const name of ['readBlocks', 'readVisible']) { +for (const name of ['readBlocks', 'readHeldBlocks', 'readVisible']) { const original = store[name] store[name] = (...args) => { reads.n++ @@ -227,6 +229,73 @@ test('a hide downloaded into a hole of the root invalidates', async (t) => { t.alike(await texts(b, group), ['depois', 'oi'], 'the hide counts') }) +// A seeder keeps a cut writer sparse (#350): below the cut, then its leaves +// and key blocks only. Read past its hole, such a copy folds like the whole +// one, and a leave arriving in that range invalidates. +test('a cut writer held sparse folds like the whole copy, and a leave past its hole invalidates (#350)', async (t) => { + const { a, b, community, group } = await world(t) + await ops.communityJoin(b.ctx, { community }) + await ops.groupJoin(b.ctx, { group }) + await ops.groupPost(b.ctx, { group, text: 'antes' }) + const wB = await ownWriter(b, group) + const cut = await coreLength(b, wB) + await ops.groupPost(b.ctx, { group, text: 'depois' }) + await ops.groupLeave(b.ctx, { group }) + const length = await coreLength(b, wB) + const diary = await store.openDiary(b.storage, b.ctx.keyPair) + try { + await store.appendRecord(diary, groupCutRecord({ group, writer: wB, cut })) + } finally { + await diary.close() + } + await copyDiary(b, a) + await copyCore(b, a, await ownWriter(b, community)) + await copyCore(b, a, wB, [[0, cut]]) + t.alike(await heldRange(a, wB, false), { contiguous: cut, length }, 'the blocks past the cut are not held') + t.alike(await memberKeys(a, group), [a.key, b.key].sort(), 'without the leave, still a member') + t.alike(await texts(a, group), ['antes', 'oi']) + t.is((await counted(a, () => memberKeys(a, group))).n, 0, 'memo warm') + await copyCore(b, a, wB, [[length - 1, length]]) + t.alike(await heldRange(a, wB, false), { contiguous: cut, length }, 'the leave alone past the hole') + const sparse = { members: await memberKeys(a, group), texts: await texts(a, group) } + t.alike(sparse.members, [a.key], 'the leave past the hole counts') + await copyCore(b, a, wB) + t.alike(await heldRange(a, wB, false), { contiguous: length, length }, 'now the whole copy') + t.alike({ members: await memberKeys(a, group), texts: await texts(a, group) }, sparse, 'the sparse copy folded like the whole one') +}) + +test('a cut community writer held sparse folds like the whole copy (#350)', async (t) => { + const { a, b, community } = await world(t) + await ops.communityJoin(b.ctx, { community }) + const cw = await ownWriter(b, community) + const cut = await coreLength(b, cw) + // A record past the cut that the cut drops, so the sparse copy has a hole. + const entry = (await store.readGroupWriters(b.storage)).find((x) => x.group === community) + const core = await store.openGroupWriter(b.storage, await b.ctx.groupKeyPair(new Uint8Array(b4a.from(entry.nonce, 'hex')))) + try { + await store.appendGroupRecord(core, communityBan({ member: a.key, on: true, cuts: [] }), { root: false, community: true }) + } finally { + await core.close() + } + await ops.communityLeave(b.ctx, { community, force: true }) + const length = await coreLength(b, cw) + t.is(length, cut + 2, 'a dropped record, then the leave past the cut') + const diary = await store.openDiary(b.storage, b.ctx.keyPair) + try { + await store.appendRecord(diary, groupCutRecord({ group: community, writer: cw, cut })) + } finally { + await diary.close() + } + const members = async () => (await ops.communityMembers(a.ctx, { community })).members.map((m) => m.member).sort() + await copyDiary(b, a) + await copyCore(b, a, cw, [[0, cut], [length - 1, length]]) + t.alike(await heldRange(a, cw, false), { contiguous: cut, length }, 'the leave alone past the hole') + const sparse = await members() + t.alike(sparse, [a.key], 'the leave past the cut counts') + await copyCore(b, a, cw) + t.alike(await members(), sparse, 'the sparse copy folded like the whole one') +}) + test('a binding diary filling a hole invalidates', async (t) => { const { a, b, community, group } = await world(t) await ops.communityJoin(b.ctx, { community }) diff --git a/packages/web-member/test/group-ops.test.js b/packages/web-member/test/group-ops.test.js index e3145d6b..2b958fb2 100644 --- a/packages/web-member/test/group-ops.test.js +++ b/packages/web-member/test/group-ops.test.js @@ -836,9 +836,11 @@ test('a group operation reads only that group\'s cores and the diaries binding t await shareGroupCore(b, a, wOther) await shareDiary(b, a) await store.writeFollows(a.storage, [b.ownHex]) - // Group cores are read with readBlocks, diaries with readAll. + // Group roots are read with readBlocks, writers with readHeldBlocks, + // diaries with readAll. const read = realStore.readAll const readBlocks = realStore.readBlocks + const readHeldBlocks = realStore.readHeldBlocks const lengths = [] realStore.readAll = async (core) => { lengths.push(core.length) @@ -848,12 +850,17 @@ test('a group operation reads only that group\'s cores and the diaries binding t lengths.push(core.length) return readBlocks(core) } + realStore.readHeldBlocks = async (core) => { + lengths.push(core.length) + return readHeldBlocks(core) + } let info try { info = await ops.groupInfo(a.ctx, { group }) } finally { realStore.readAll = read realStore.readBlocks = readBlocks + realStore.readHeldBlocks = readHeldBlocks } t.is(info.members, 1, 'B\'s diary binds no core of this group here') // The own diary and this group's root: never B's diary nor the writer From b546e77215d0dfe23065ad44f52da78930ec9dba Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 00:43:29 -0300 Subject: [PATCH 04/25] feat(cli): read a listed group writer past its first hole The account service reads the writers a seeder lists as the nodes do: past the first hole, the leaves and key blocks held, so a cut writer the seeder keeps sparse folds like the whole copy (#350). --- apps/cli/lib/account/pages-groups.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/cli/lib/account/pages-groups.js b/apps/cli/lib/account/pages-groups.js index c696b96a..03033c97 100644 --- a/apps/cli/lib/account/pages-groups.js +++ b/apps/cli/lib/account/pages-groups.js @@ -45,7 +45,7 @@ const hcrypto = require('hypercore-crypto') const Hypercore = require('hypercore') const Corestore = require('corestore') const { WebSocket, createWebSocketStream } = require('ws') -const { writeJsonAtomic, appendRecord, readAll, readBlocks, openReplica, purgeReplica } = require('@varanda/core/lib/store') +const { writeJsonAtomic, appendRecord, readAll, readBlocks, readHeldBlocks, openReplica, purgeReplica } = require('@varanda/core/lib/store') const { refuseSymlinkDir, certForKey, seederCertForKey, isPlainObject } = require('@varanda/core/lib/cert') const { attachAdmission } = require('@varanda/core/lib/admission') const { attachGroups, GROUPS_LIMIT_MAX } = require('@varanda/core/lib/groups-net') @@ -711,7 +711,9 @@ function createPagesGroups ({ dir, writer, pagesStore, keys, association, blockl if (!got.complete) incomplete = true const core = got.core if (core.fork !== 0) continue - const entries = await readBlocks(core) + // A seeder keeps a cut writer sparse (#350): past its first hole, its + // leaves and key blocks. + const entries = await readHeldBlocks(core) const h = entries[0] && entries[0].record if (groups.validateWriterHeader(h) !== null || h.group !== key) continue logs.push({ key: it.key, diary: h.diary, entries, length: core.contiguousLength, signed: core.length }) From 67b799840e81538feab677553bfb4cb3839d970a Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 01:08:29 -0300 Subject: [PATCH 05/25] feat(seeder): keep a cut writer sparse and a released one as a stub A writer its diary cut keeps its blocks below the cut and, past it, only its leaves and key blocks: the rest is fetched a chunk at a time, read and cleared, and the quota counts what is kept. A released writer is a stub (block 0, leaves, key blocks), also a replaced boxless writer of a private group, which readers then reject by its header. The binding persists the cut and the range read before clearing; a restart recounts it, a restored cut clears nothing new until the scan confirms it, and a cut that is gone fetches again what clearing removed. A lower cut lifts a refused join once; a writer dropped for what it keeps is refused like one at admission (#350). --- packages/core/lib/seeder.js | 425 ++++++++++++++++-- packages/core/test/seeder-groups-cuts.test.js | 255 +++++++++-- 2 files changed, 603 insertions(+), 77 deletions(-) diff --git a/packages/core/lib/seeder.js b/packages/core/lib/seeder.js index ccf28540..00d6cf68 100644 --- a/packages/core/lib/seeder.js +++ b/packages/core/lib/seeder.js @@ -90,16 +90,21 @@ // diary that count are the ones its slot walk counts (lib/group.js // slotWalk, over its group-join and group-cut records): at most // MAX_GROUP_WRITERS_PER_DIARY live and MAX_FROZEN_WRITERS_PER_DIARY frozen, -// plus each of the up to MAX_WRITER_RELEASES_PER_DIARY writers it released -// (liveGroupJoins). Listed in admitted.json (`groups: { K: { diary, group, -// seq } }`), persisted before the open, purged before un-listing. +// plus a stub of each of the up to MAX_WRITER_RELEASES_PER_DIARY writers it +// released (liveGroupJoins). Listed in admitted.json (`groups: { K: { diary, +// group, seq, cut, clearedFrom, clearedTo } }`), persisted before the open, +// purged before un-listing. // Leaving (a join turned off) NEVER purges: it only marks the writer as // left in the group index (lib/group-index.js), so the moderation record of // a departed member stays verifiable, and a counted writer whose join is -// off is still admitted. A writer its diary cut (frozen or released) is -// held whole, like any writer: a reader folds from a contiguous copy, so -// nothing past the cut is cleared yet. Purges happen for a dropped or blocked diary, a root its blocked or revoked -// creator bound, a root key in the blocklist snapshot, a fork or the +// off is still admitted. A writer its diary cut keeps its blocks below the +// cut and, past it, only its group-leave and group-key blocks, which the +// fold still counts and readers read past a hole (_cutGroupCore): the rest +// is fetched, read and cleared, and the quota counts what is kept. A writer +// cut at 0 (released) is such a stub: block 0, its leaves and its key +// blocks, also a replaced boxless writer of a private group, which readers +// then reject by its header. Purges happen for a dropped or blocked diary, +// a root its blocked or revoked creator bound, a root key in the blocklist snapshot, a fork or the // quota. // Group keys never travel on the admission channel; members learn writer // sets over 'varanda/groups/1' (lib/groups-net.js) and the /index routes. @@ -632,7 +637,7 @@ function mediaBinding (value) { // creator left for another reason (quota, its own drop). function groupBinding (value) { if (!isPlainObject(value)) return null - const { diary, group, seq, by, bound, issuer } = value + const { diary, group, seq, by, bound, issuer, cut, clearedFrom, clearedTo } = value if (typeof diary !== 'string' || !HEX64.test(diary)) return null if (typeof group !== 'string' || !HEX64.test(group)) return null if (!Number.isSafeInteger(seq) || seq < 1) return null @@ -640,6 +645,17 @@ function groupBinding (value) { if (typeof by === 'string' && HEX64.test(by)) out.by = by if (bound === true) out.bound = true if (out.bound === true && typeof issuer === 'string' && HEX64.test(issuer)) out.issuer = issuer + // A writer its diary cut is kept below `cut` and, in [clearedFrom, + // clearedTo), only its leaves and key blocks (_cutGroupCore); cut 0 is a + // released writer's stub. A range that does not fit the cut reads as + // nothing cleared yet. + if (Number.isSafeInteger(cut) && cut >= 0) { + const below = keptBelow(cut) + const fits = Number.isSafeInteger(clearedFrom) && Number.isSafeInteger(clearedTo) && below <= clearedFrom && clearedFrom <= clearedTo + out.cut = cut + out.clearedFrom = fits ? clearedFrom : below + out.clearedTo = fits ? clearedTo : below + } return out } @@ -793,8 +809,11 @@ async function localMediaVerdict (core, diaryHex) { // writer's `box` must match it, hex64 in a private group and null in a // public one, else 'header-invalid' as in the fold and member sync. A // version 1 header (the validators' 'legacy') is 'header-invalid' too: no -// wire reason of its own, and the startup re-gate purges it. -function groupVerdict (raw, key, group, diary, visibility = null) { +// wire reason of its own, and the startup re-gate purges it. `released`: the +// writer's diary cut it at 0, so a boxless writer of a private group (one +// its diary replaced) is kept as a stub, which readers fetch and reject by +// its header instead of waiting for it (#350). +function groupVerdict (raw, key, group, diary, visibility = null, released = false) { try { const header = decodeBlock(raw) if (header === null) return 'header-invalid' @@ -805,7 +824,7 @@ function groupVerdict (raw, key, group, diary, visibility = null) { if (validateWriterHeader(header) !== null) return 'header-invalid' if (header.group !== group) return 'wrong-group' if (header.diary !== diary) return 'wrong-diary' - if (visibility === 'private' && header.box === null) return 'header-invalid' + if (visibility === 'private' && header.box === null && !released) return 'header-invalid' if (visibility === 'public' && header.box !== null) return 'header-invalid' return null } catch { @@ -818,18 +837,18 @@ function groupVerdict (raw, key, group, diary, visibility = null) { async function localGroupVerdict (core, key, binding, visibility = null) { try { const raw = await core.get(0, { wait: false, raw: true }) - return { reason: groupVerdict(raw, key, binding.group, binding.diary, visibility), raw } + return { reason: groupVerdict(raw, key, binding.group, binding.diary, visibility, key !== binding.group && binding.cut === 0), raw } } catch { return { reason: 'header-invalid', raw: null } } } // Writer cores one diary may hold at a seeder in one group, left ones -// included (leaving never frees a slot): its live and frozen slots, and -// each released writer, held whole (#350). +// included (leaving never frees a slot): its live and frozen slots, and a +// stub of each released writer (#350). const MAX_GROUP_HELD_PER_DIARY = MAX_GROUP_WRITERS_PER_DIARY + MAX_FROZEN_WRITERS_PER_DIARY + MAX_WRITER_RELEASES_PER_DIARY // And across all its groups: its share of the group cores, each released -// writer, held whole, included. It stays at the live and frozen slots of +// writer's stub included. It stays at the live and frozen slots of // every group (owner decision). const MAX_GROUP_CORES_PER_DIARY = MAX_GROUPS_PER_DIARY * (MAX_GROUP_WRITERS_PER_DIARY + MAX_FROZEN_WRITERS_PER_DIARY) // Waits before a group core that timed out or found no peer is tried again @@ -915,6 +934,63 @@ function groupWalkStep (walks, group, e) { slotWalkStep(s, e, group) } +// The cut a diary's walk for G gives writer W: 0 once released (or cut at 0 +// with no release left), its lowest cut while counted, null with none. +function groupWalkCut (walks, group, writer) { + const s = walks.get(group) + if (s === undefined || writer === group) return null + const st = s.state.get(writer) + if (st === 'released') return 0 + if ((st === 'live' || st === 'frozen') && s.cut.has(writer)) return s.cut.get(writer) + return null +} + +// The blocks of a writer cut at `cut` a seeder keeps in full: those below +// it, block 0 always. +function keptBelow (cut) { + return Math.max(cut, 1) +} + +// Whether a block past a writer's cut stays at the seeder: a group-leave or +// a group-key, which the fold keeps past a cut (lib/group.js bindCores) and +// readers read past a hole (store.js readHeldBlocks). +function keptPastCut (raw) { + if (raw === null || raw === undefined) return false + try { + const block = decodeBlock(raw) + return block !== null && (block.type === 'group-leave' || block.type === 'group-key') + } catch { + return false + } +} + +// Blocks past a cut read per step of _cutGroupCore: what a seeder stores +// beyond what it keeps stays within one chunk. +const CUT_CHUNK = 256 + +// The held blocks of [from, to) a cut keeps, as [seq, bytes], and the runs +// [start, end) of the others, to clear. +async function cutRange (core, from, to) { + const keep = [] + const clear = [] + let start = -1 + for (let i = from; i < to; i++) { + let raw = null + try { + raw = await core.get(i, { wait: false, raw: true }) + } catch {} + if (raw !== null && keptPastCut(raw)) { + keep.push([i, raw.byteLength]) + if (start !== -1) clear.push([start, i]) + start = -1 + } else if (raw !== null && start === -1) { + start = i + } + } + if (start !== -1) clear.push([start, to]) + return { keep, clear } +} + // blocklist.json as stored, unwrapped ({ v: 1, snapshot }: store.js // writeBlocklist), or null. function readSnapshot (dataDir, options) { @@ -1790,7 +1866,7 @@ class Seeder { core = await openMediaReplica(this.store, key) if (core.fork !== 0) reason = 'fork' else if (core.contiguousLength === 0) reason = 'no-header' - else if (this._overQuota(core)) reason = 'quota' + else if (this._groupOverQuota(key === binding.group || binding.cut === undefined ? null : binding.cut, core)) reason = 'quota' else ({ reason, raw } = await localGroupVerdict(core, key, binding, this._groupVisibility(binding.group))) } if (reason === null && (groupSurvivors.length >= this.maxGroupCores || (perGroup.get(binding.group) || 0) >= this.maxGroupWriters)) reason = 'group-full' @@ -1848,7 +1924,7 @@ class Seeder { this._groupIndex.setHeader(key, decodeBlock(raw)) this._keepGroupHeader(key, decodeBlock(raw)) } - this._serveGroup(key, binding, core, { headerOk: true }) + this._serveGroup(key, binding, core, { headerOk: true, restored: true }) if (key === binding.group && !this._served.has(binding.diary)) this._indexKeptRoot(key, false) } // A writer served after its root's header was first recorded in this @@ -2223,6 +2299,15 @@ class Seeder { return length > this.maxLogBlocks || byteLength > this.maxLogBytes } + // A group core's log quota at admission and restore: the whole log's or, + // for a writer cut at `cut`, the blocks it keeps below the cut (its bytes + // and the blocks it keeps past the cut count as they are read, + // _cutGroupCore). + _groupOverQuota (cut, { length, byteLength }) { + if (cut === null) return this._overQuota({ length, byteLength }) + return Math.min(keptBelow(cut), length) > this.maxLogBlocks + } + _serve (key, core, { regate, cert0 = null, pushFrom = core.length }) { // Media scan state (header comment): `scanned` is the next block to read // for media-core records (block 0 is the certificate), `decls` the @@ -2880,7 +2965,14 @@ class Seeder { } if (block.type === 'group-cut') { const r = validRecordCopy(block, i) - if (r !== null) groupWalkStep(entry.groupWalks, r.group, { kind: 'cut', writer: r.writer, cut: r.cut, seq: i }) + if (r === null) continue + const before = groupWalkCut(entry.groupWalks, r.group, r.writer) + groupWalkStep(entry.groupWalks, r.group, { kind: 'cut', writer: r.writer, cut: r.cut, seq: i }) + const after = groupWalkCut(entry.groupWalks, r.group, r.writer) + // A lower cut keeps less (the quota) and a release keeps a boxless + // writer (groupVerdict): a join refused before is tried again, once + // per lower cut. + if (after !== null && (before === null || after < before)) entry.groupRefused.delete(r.writer) continue } // Folded here, not in _runIndex: that queue pauses at capacity and @@ -4026,6 +4118,9 @@ class Seeder { if (entry.dropped || !entry.gated || this._closing) return for (const [k, b] of [...this._groups]) { if (b === null || b.diary !== diary) continue + // The cut its walk gives a writer, also while its core still opens + // (_serveGroup reads the binding's). + if (k !== b.group) this._groupCutScan(k, b, entry) if (!this._groupServed.has(k)) continue const live = entry.groupLive.get(k) const bound = live !== undefined && live.group === b.group && !live.left @@ -4075,6 +4170,70 @@ class Seeder { } } + // A lower cut from the walk applies at once: the scan only adds marks, so + // the diary's cut is never above it. A higher one, or none, only once the + // scan caught up with the diary (a cut restored from admitted.json whose + // block can no longer be read, for example), and what clearing removed is + // fetched again. + _groupCutScan (key, b, entry) { + const cut = groupWalkCut(entry.groupWalks, b.group, key) + const now = Number.isSafeInteger(b.cut) ? b.cut : null + const caught = scanCaughtUp(entry) + if ((cut !== null && (now === null || cut < now)) || (caught && cut !== now)) return this._applyGroupCut(key, b, cut) + const served = this._groupServed.get(key) + if (caught && served && !served.dropped && !served.cutConfirmed) { + served.cutConfirmed = true + if (served.cut !== null) this._cutGroupCore(key, served) + } + } + + // Writer K's cut (null: none) on its binding and, once served, on its + // entry: the download stops at the new cut, a lower cut reads the blocks + // now past it (_cutGroupCore), and a higher one or none fetches again what + // clearing removed below it. + _applyGroupCut (key, b, cut) { + const old = Number.isSafeInteger(b.cut) ? b.cut : null + if (cut === null) { + delete b.cut + delete b.clearedFrom + delete b.clearedTo + } else { + const below = keptBelow(cut) + if (old === null) { + b.clearedFrom = below + b.clearedTo = below + } else if (cut > old) { + b.clearedFrom = Math.max(b.clearedFrom, below) + b.clearedTo = Math.max(b.clearedTo, b.clearedFrom) + } + b.cut = cut + } + this._persist().catch(noop) + const entry = this._groupServed.get(key) + if (!entry || entry.dropped) return + entry.cut = cut + entry.cutConfirmed = true + entry.belowBytes = 0 + entry.belowCounted = 0 + for (const [i, bytes] of entry.keptPast) { + if (cut !== null && i >= b.clearedFrom) continue + entry.keptPast.delete(i) + entry.keptPastBytes -= bytes + } + this._rearmGroupDownload(entry) + // A boxless writer kept as a stub is judged again once no longer + // released. + if (old === 0 && cut !== 0) { + const visibility = this._groupVisibility(entry.group) + if (visibility !== null) this._recheckGroupBoxes(entry.group, visibility) + } + if (cut === null) { + if (this._overQuota(entry.core)) this._dropGroup(key, 'quota') + return + } + this._cutGroupCore(key, entry) + } + _retryGroups (peer) { if (peer) { if (!peer.socket || peer.seederDropped) return @@ -4285,7 +4444,8 @@ class Seeder { return this._rejectGroup(key, diary, group, 'timeout', seq) } if (!this._groupStillWanted(diary, entry, key, group)) return - const verdict = groupVerdict(res.raw, key, group, key === group ? null : diary, this._groupVisibility(group)) + const cut = key === group ? null : groupWalkCut(entry.groupWalks, group, key) + const verdict = groupVerdict(res.raw, key, group, key === group ? null : diary, this._groupVisibility(group), cut === 0) if (verdict !== null) { entry.groupRefused.set(key, seq) return this._rejectGroup(key, diary, group, verdict, seq) @@ -4295,8 +4455,8 @@ class Seeder { if (key === group) this._keepGroupHeader(group, decodeBlock(res.raw)) // Refusals after a fetch are remembered (this join) or backed off, so a // rescan does not fetch the same core again. A writer its diary cut is - // held whole, so its whole log is judged. - if (this._overQuota(res)) { + // judged on the blocks it keeps below the cut. + if (this._groupOverQuota(cut, res)) { entry.groupRefused.set(key, seq) return this._rejectGroup(key, diary, group, 'quota', seq) } @@ -4338,6 +4498,7 @@ class Seeder { if (wasGone.issuer !== null) rootBinding.issuer = wasGone.issuer } const writerBinding = { diary: owner, group, seq: ownSeq } + if (cut !== null) Object.assign(writerBinding, { cut, clearedFrom: keptBelow(cut), clearedTo: keptBelow(cut) }) this._groups.set(key, key === group ? rootBinding : writerBinding) let core let opening = false @@ -4370,7 +4531,7 @@ class Seeder { else if (key === group && this._rootsClosed()) late = 'denials-full' else if (!this._groupStillWanted(diary, entry, key, group)) late = 'unbound' else if (core.fork !== 0) late = 'fork' - else if (this._overQuota(core)) late = 'quota' + else if (this._groupOverQuota(key === group || writerBinding.cut === undefined ? null : writerBinding.cut, core)) late = 'quota' if (late === null && this._closing) return if (late !== null) { await core.close().catch(noop) @@ -4408,7 +4569,9 @@ class Seeder { // Serving a group core: live download, fork watcher, the log quota on // every append, no growth broadcast. `expected` and `headerOk` as for // media (_serveMedia): a fresh admission is re-gated once block 0 is local. - _serveGroup (key, { diary, group }, core, { expected = null, headerOk = false, by = diary } = {}) { + _serveGroup (key, { diary, group }, core, { expected = null, headerOk = false, by = diary, restored = false } = {}) { + const binding = this._groups.get(key) + const cut = key !== group && binding && Number.isSafeInteger(binding.cut) ? binding.cut : null const entry = { core, diary, @@ -4427,16 +4590,50 @@ class Seeder { rootScanning: false, rootDecoded: 0, // blocks a root scan decoded (past the group-meta prefilter) rootRetry: null, // the retry of a failed root block read (_retryRootScan) - rootReadFailures: 0 + rootReadFailures: 0, + // A writer its diary cut (binding `cut`): its blocks below the cut and, + // in the range its binding lists as read, only its leaves and key + // blocks (`keptPast`, seq -> bytes; _cutGroupCore). `cutConfirmed` + // once the diary's scan gave the cut: one restored from admitted.json + // clears nothing new before. `belowBytes` the bytes of the blocks below + // the cut counted so far (`belowCounted` of them), `counted` once the + // range read was read again in this serve, `past` the fetch of blocks + // past the cut being read, `cutting`, `cutAgain` and `wake` one pass at + // a time. + cut, + cutConfirmed: !restored, + keptPast: new Map(), + keptPastBytes: 0, + belowBytes: 0, + belowCounted: 0, + counted: false, + past: null, + cutting: false, + cutAgain: false, + wake: null } const checkQuota = () => { if (entry.dropped || this._closing) return + if (entry.cut !== null) return this._cutGroupCore(key, entry) if (this._overQuota(core)) this._dropGroup(key, 'quota') } + // A block below a cut counts toward the bytes kept, read only while the + // whole log is over the byte quota. + const onDownload = (index) => { + if (entry.cut !== null && index < keptBelow(entry.cut) && core.byteLength > this.maxLogBytes) this._cutGroupCore(key, entry) + } core.on('append', checkQuota) - entry.stopQuota = () => core.off('append', checkQuota) + core.on('download', onDownload) + entry.stopQuota = () => { + core.off('append', checkQuota) + core.off('download', onDownload) + try { + if (entry.past) entry.past.range.destroy() + } catch {} + if (entry.wake !== null) entry.wake() + } entry.stopFork = watchFork(core, (reason) => this._dropGroup(key, reason)) - entry.download = core.download({ start: 0, end: -1 }) + entry.download = core.download(cut === null ? { start: 0, end: -1 } : { start: 0, end: keptBelow(cut) }) entry.stopGrow = watchContiguous(core, () => { this._gateGroup(key, entry) this._scanRoot(key, entry) @@ -4460,6 +4657,173 @@ class Seeder { if (!entry.headerOk && core.contiguousLength >= 1) this._gateGroup(key, entry) } + // The download of a served writer for its cut: the blocks below it, block + // 0 always, or the whole log without one. A block a pass cleared under a + // cut that rose meanwhile is fetched again. + _rearmGroupDownload (entry) { + try { + if (entry.download) entry.download.destroy() + if (entry.past) entry.past.range.destroy() + } catch {} + entry.past = null + entry.download = entry.core.download(entry.cut === null ? { start: 0, end: -1 } : { start: 0, end: keptBelow(entry.cut) }) + } + + // Whether a cut writer keeps more than the log quota: the blocks below its + // cut, block 0 always, and the leaves and key blocks kept past it, and + // their bytes. Counted only once the whole log is over the quota. + _groupKeptOver (entry) { + const core = entry.core + if (!this._overQuota(core)) return false + const blocks = Math.min(keptBelow(entry.cut), core.length) + entry.keptPast.size + return blocks > this.maxLogBlocks || entry.belowBytes + entry.keptPastBytes > this.maxLogBytes + } + + // Waits for blocks [from, to) of a cut writer, fetched by one range kept + // across passes: true once all are local, false when another pass was + // asked for first or the fetch ended without them. + async _groupPastRange (entry, from, to) { + let p = entry.past + if (p !== null && (p.from !== from || p.to !== to || p.failed)) { + try { + p.range.destroy() + } catch {} + p = null + } + if (p === null) { + const range = entry.core.download({ start: from, end: to }) + const q = { from, to, range, done: false, failed: false, settled: null } + q.settled = range.done().then(() => { q.done = true }, () => { q.failed = true }) + entry.past = q + p = q + } + if (!p.done && !p.failed) { + await new Promise((resolve) => { + entry.wake = resolve + p.settled.then(resolve) + }) + entry.wake = null + } + if (!p.done) return false + if (entry.past === p) entry.past = null + return true + } + + // A writer its diary cut (docs/protocol.md section 10) keeps its blocks + // below the cut and, from the cut on, its group-leave and group-key blocks + // only, which every fold still counts there and readers read past a hole: + // a reader fed by seeders folds as one holding the whole log. The blocks + // past the cut are fetched a chunk at a time, read, and cleared unless + // they are one of those (Hypercore keeps the tree nodes the proofs of the + // kept blocks need). A chunk is listed as read before its blocks go: after + // a crash in between, the next serve reads the range again and clears + // what it still holds beyond the kept blocks. The quota counts what is + // kept. One pass at a time; growth or a new cut during a pass runs another. + _cutGroupCore (key, entry) { + if (entry.cutting) { + entry.cutAgain = true + if (entry.wake !== null) entry.wake() + return + } + entry.cutting = true + const stale = () => entry.dropped || this._closing || this._groupServed.get(key) !== entry + const run = async () => { + for (;;) { + entry.cutAgain = false + if (stale()) return + const cut = entry.cut + const b = this._groups.get(key) + if (cut === null || !b || !Number.isSafeInteger(b.clearedFrom)) return + const core = entry.core + const below = keptBelow(cut) + const moved = () => stale() || entry.cut !== cut + if (!entry.counted) { + const { keep, clear } = await cutRange(core, b.clearedFrom, b.clearedTo) + if (moved()) continue + for (const [start, end] of clear) await core.clear(start, end) + if (moved()) { + this._rearmGroupDownload(entry) + continue + } + entry.keptPast = new Map(keep) + entry.keptPastBytes = keep.reduce((n, [, bytes]) => n + bytes, 0) + entry.counted = true + } + if (core.byteLength > this.maxLogBytes) { + const to = Math.min(core.contiguousLength, below) + while (entry.belowCounted < to) { + let raw = null + try { + raw = await core.get(entry.belowCounted, { wait: false, raw: true }) + } catch {} + if (moved()) break + entry.belowBytes += raw === null ? 0 : raw.byteLength + entry.belowCounted++ + } + if (moved()) continue + } + if (this._groupKeptOver(entry)) { + // Refused as at admission, or every rescan would fetch it again. + const owner = this._served.get(entry.diary) + if (owner && !owner.dropped) owner.groupRefused.set(key, b.seq) + this._dropGroup(key, 'quota') + return + } + if (!entry.cutConfirmed) return + // A lower cut first reads down from the range already read, so that + // range stays one piece; then up to the signed length. + const end = Math.min(core.length, MAX_LOG_BLOCKS) + const down = b.clearedFrom > below + let from + let to + if (down) { + from = Math.max(below, b.clearedFrom - CUT_CHUNK) + to = b.clearedFrom + } else if (b.clearedTo < end) { + from = b.clearedTo + to = Math.min(end, from + CUT_CHUNK) + } else { + return + } + const held = await this._groupPastRange(entry, from, to) + if (moved()) continue + if (!held) { + if (entry.cutAgain) continue + return + } + const { keep, clear } = await cutRange(core, from, to) + if (moved()) continue + if (down) b.clearedFrom = from + else b.clearedTo = to + try { + await this._persist() + } catch (err) { + entry.counted = false + throw err + } + if (moved()) { + entry.counted = false + continue + } + for (const [start, end] of clear) await core.clear(start, end) + if (moved()) { + entry.counted = false + this._rearmGroupDownload(entry) + continue + } + for (const [i, bytes] of keep) { + if (entry.keptPast.has(i)) continue + entry.keptPast.set(i, bytes) + entry.keptPastBytes += bytes + } + } + } + run().catch(noop).then(() => { + entry.cutting = false + if (entry.cutAgain && !stale()) this._cutGroupCore(key, entry) + }) + } + // Local re-gate of a served group core, the twin of _gateMedia: block 0 // in the main store must be byte-identical to the quarantined one (else // 'fork') and a header naming the same keys. @@ -4478,7 +4842,7 @@ class Seeder { if (raw === null) return 'header-invalid' if (entry.expected !== null && !b4a.equals(raw, entry.expected)) return 'fork' visibility = this._groupVisibility(entry.group) - return groupVerdict(raw, key, entry.group, entry.diary, visibility) + return groupVerdict(raw, key, entry.group, entry.diary, visibility, entry.cut === 0) } check().then(async (reason) => { if (this.regateHook) await this.regateHook(key, reason) @@ -4488,7 +4852,7 @@ class Seeder { // _recheckGroupBoxes skips a writer not gated yet: check the box // against the visibility known now. const now = this._groupVisibility(entry.group) - if (reason === null && now !== visibility) reason = groupVerdict(raw, key, entry.group, entry.diary, now) + if (reason === null && now !== visibility) reason = groupVerdict(raw, key, entry.group, entry.diary, now, entry.cut === 0) if (reason !== null) return this._dropGroup(key, reason) entry.headerOk = true entry.expected = null @@ -4684,7 +5048,7 @@ class Seeder { raw = await entry.core.get(0, { wait: false, raw: true }) } catch {} if (raw === null) return null - return groupVerdict(raw, key, entry.group, entry.diary, visibility) + return groupVerdict(raw, key, entry.group, entry.diary, visibility, entry.cut === 0) } check().then((reason) => { if (reason === null || entry.dropped || this._closing || this._groupServed.get(key) !== entry) return @@ -8340,5 +8704,6 @@ module.exports = { DISCOVERY_VIEW_MAX_KEYS, // For tests. liveGroupJoins, - groupWalkStep + groupWalkStep, + groupWalkCut } diff --git a/packages/core/test/seeder-groups-cuts.test.js b/packages/core/test/seeder-groups-cuts.test.js index 5e2e40ca..a533011c 100644 --- a/packages/core/test/seeder-groups-cuts.test.js +++ b/packages/core/test/seeder-groups-cuts.test.js @@ -1,17 +1,19 @@ 'use strict' // The writers a diary cut (#350, docs/protocol.md section 10) at a seeder: -// the slot walk picks the writers that count (liveGroupJoins), and a frozen -// or released writer is held whole, as a reader folds from a contiguous -// copy (a reduced copy comes with the client nodes' sparse read). +// the slot walk picks the writers that count (liveGroupJoins), a frozen +// writer is kept below its cut plus its leaves and key blocks, a released +// one as a stub (block 0, leaves, key blocks), the quota counts what is +// kept, and a restart recomputes both from admitted.json and the scan. const test = require('brittle') const b4a = require('b4a') const { testnet, until } = require('./helpers/net') -const { liveGroupJoins, groupWalkStep } = require('../lib/seeder') +const { liveGroupJoins, groupWalkStep, groupWalkCut } = require('../lib/seeder') const { groupJoinRecord, groupCutRecord, + deleteRecord, MAX_GROUP_WRITERS_PER_DIARY, MAX_FROZEN_WRITERS_PER_DIARY, MAX_WRITER_RELEASES_PER_DIARY, @@ -32,7 +34,8 @@ const { rejected, downloaded, readGroups, - scansDone + scansDone, + gate } = require('./helpers/seeder-groups') const G = 'aa'.repeat(32) @@ -78,6 +81,9 @@ test('liveGroupJoins: a cut frees a slot for a 9th writer, never past 8 live, 8 t.ok(released.has(key(8)), 'so does a release') t.alike(released.get(key(0)), { group: G, seq: 1, left: true }, 'and the released writer stays, left') t.is(diary([...eight, cut(key(0), 0)]).walks.get(G).state.get(key(0)), 'released') + t.is(groupWalkCut(diary([...eight, cut(key(0), 0)]).walks, G, key(0)), 0, 'kept at cut 0') + t.is(groupWalkCut(diary([...eight, cut(key(1), 2)]).walks, G, key(1)), 2, 'a frozen writer at its cut') + t.is(groupWalkCut(diary(eight).walks, G, key(0)), null, 'a live writer has none') // Eight frozen, then eight live: the 17th never counts. const full = [] @@ -229,7 +235,7 @@ async function blocksHeld (core, from, to) { return out } -test('a cut writer is held whole: its blocks past the cut stay, and a fresh reader reads its leave in order (#350)', async (t) => { +test('a cut after the download clears from the cut, keeps only leaves and key blocks past it, and serves what it keeps with proofs (#350)', async (t) => { t.timeout(150000) const { bootstrap } = await testnet(t) const assoc = makeAssociation() @@ -241,32 +247,43 @@ test('a cut writer is held whole: its blocks past the cut stay, and a fresh read await admit(B, assoc) await admitted(events, W) await downloaded(seeder, W, 6) + t.absent('cut' in readGroups(dataDir)[W], 'no cut yet') - // B cuts W at 3, then keeps writing to it: a post, a leave, another post. await B.diary.append(groupCutRecord({ group: R, writer: W, cut: 3 })) - await settledFor(seeder, B) + const entry = seeder._groupServed.get(W) + await until(async () => entry.cut === 3 && !entry.cutting && seeder._groups.get(W).clearedTo === 6 && (await blocksHeld(entry.core, 0, 6)).length === 3, { name: 'W cortado em 3' }) + t.alike(await blocksHeld(entry.core, 0, 6), [0, 1, 2], 'only the blocks below the cut are stored') + t.is(entry.core.contiguousLength, 3) + t.alike(readGroups(dataDir)[W], { diary: B.key, group: R, seq: 1, cut: 3, clearedFrom: 3, clearedTo: 6 }, 'the cut and the range read are persisted with the binding') + t.ok(seeder._groupServed.has(W), 'the writer stays served') + + // B keeps writing past its cut: a post goes, a leave and a key block stay + // (the fold counts them past a cut), and so does the next post's gap. await B.cores.w.append(b4a.from(JSON.stringify(groupPost({ text: 'depois do corte' })))) await B.cores.w.append(b4a.from(JSON.stringify(groupLeave({ cuts: [{ writer: W, seq: 7 }] })))) + await B.cores.w.append(b4a.from(JSON.stringify({ type: 'group-key' }))) await B.cores.w.append(b4a.from(JSON.stringify(groupPost({ text: 'mais um' })))) - await downloaded(seeder, W, 9) - const entry = seeder._groupServed.get(W) - t.alike(await blocksHeld(entry.core, 0, 9), [0, 1, 2, 3, 4, 5, 6, 7, 8], 'every block stays, past the cut too') - t.absent('cut' in readGroups(dataDir)[W], 'no cut is kept with the binding') + await until(async () => entry.core.length === 10 && !entry.cutting && seeder._groups.get(W).clearedTo === 10, { name: 'W lido depois do corte' }) + t.alike(await blocksHeld(entry.core, 0, 10), [0, 1, 2, 7, 8], 'below the cut, then the leave and the key block only') + t.is(entry.keptPast.size, 2, 'the quota counts the two blocks kept past the cut') - // B goes; a reader that dials the seeder only reads W from block 0 up, - // as the nodes do (store.js readBlocks stops at the first hole). + // B goes; a reader that dials the seeder only verifies the kept blocks + // against the signed length (Hypercore checks each proof). await B.swarm.destroy() const r = await reader(t, bootstrap, publicKey) const copy = r.store.get({ key: b4a.from(W, 'hex') }) await copy.ready() - await until(() => copy.length === 9, { name: 'comprimento assinado no leitor', timeoutMs: 30000 }) - const types = [] - for (let i = 0; i < 9; i++) types.push(JSON.parse(b4a.toString(await copy.get(i, { timeout: 15000 }))).type) - t.is(types[7], 'group-leave', 'the leave past the cut is reached by a contiguous read') + await until(() => copy.length === 10, { name: 'comprimento assinado no leitor', timeoutMs: 30000 }) + for (let i = 0; i < 3; i++) t.ok(await copy.get(i, { timeout: 15000 }), 'block ' + i + ' verified from the seeder') + const leave = JSON.parse(b4a.toString(await copy.get(7, { timeout: 15000 }))) + t.is(leave.type, 'group-leave', 'the leave past the cut is served, verified') + t.is(JSON.parse(b4a.toString(await copy.get(8, { timeout: 15000 }))).type, 'group-key', 'and the key block') + await t.exception(copy.get(3, { timeout: 3000 }), 'a post past the cut is not served') + await t.exception(copy.get(9, { timeout: 3000 }), 'nor one after the leave') t.is(events.filter((e) => e.event === 'fatal').length, 0) }) -test('a 9th writer after a freeze is admitted, a counted writer whose join is off is admitted as left, the 10th never (#350)', async (t) => { +test('a 9th writer after a freeze is admitted at its cut, a counted writer whose join is off is admitted as left, the 10th never (#350)', async (t) => { t.timeout(240000) const { bootstrap } = await testnet(t) const assoc = makeAssociation() @@ -296,9 +313,12 @@ test('a 9th writer after a freeze is admitted, a counted writer whose join is of await settledFor(seeder, B) t.absent(events.some((e) => e.key === W[9]), 'the 10th is past the cap and never tried') t.absent(W[9] in readGroups(dataDir)) - await downloaded(seeder, W[0], 2) - t.pass('the frozen writer is held whole') + t.is(readGroups(dataDir)[W[0]].cut, 1, 'the frozen writer is listed at its cut') + await until(() => seeder._groups.get(W[0]).clearedTo === 2 && !seeder._groupServed.get(W[0]).cutting, { name: 'W0 lido depois do corte' }) + t.is(seeder._groupServed.get(W[0]).core.contiguousLength, 1, 'W0 below its cut') + t.is(await seeder._groupServed.get(W[0]).core.has(1), false, 'and holds nothing past it') await downloaded(seeder, W[8], 2) + t.absent('cut' in readGroups(dataDir)[W[8]], 'the 9th has no cut') const rows = new Map(seeder._groupIndex.page(R).items.filter((i) => i.diary === B.key).map((i) => [i.key, i])) t.is(rows.size, 9, 'nine writers of B listed') @@ -308,7 +328,7 @@ test('a 9th writer after a freeze is admitted, a counted writer whose join is of t.is(events.filter((e) => e.event === 'fatal').length, 0) }) -test('a writer its diary released (cut 0) stays held whole, and a fresh reader reads its leave in order (#350)', async (t) => { +test('a writer its diary released (cut 0) is kept as a stub that serves its leave to a fresh reader (#350)', async (t) => { t.timeout(150000) const { bootstrap } = await testnet(t) const assoc = makeAssociation() @@ -325,28 +345,28 @@ test('a writer its diary released (cut 0) stays held whole, and a fresh reader r await downloaded(seeder, W, 8) await B.diary.append(groupCutRecord({ group: R, writer: W, cut: 0 })) - await settledFor(seeder, B) const entry = seeder._groupServed.get(W) - t.ok(entry, 'still served') - t.alike(await blocksHeld(entry.core, 0, 8), [0, 1, 2, 3, 4, 5, 6, 7], 'every block stays') - t.absent('cut' in readGroups(dataDir)[W], 'no cut is kept with the binding') - t.ok(seeder._groupIndex.writer(W), 'it stays in the index') + await until(async () => entry.cut === 0 && !entry.cutting && seeder._groups.get(W).clearedTo === 8 && (await blocksHeld(entry.core, 0, 8)).length === 2, { name: 'W reduzido a um toco' }) + t.alike(await blocksHeld(entry.core, 0, 8), [0, 6], 'block 0 and the leave only') + t.is(readGroups(dataDir)[W].cut, 0, 'the stub is persisted at cut 0') + t.ok(seeder._groupIndex.writer(W), 'and stays in the index') t.absent(events.some((e) => e.event === 'group-rejected' && e.key === W), 'never purged') + await settledFor(seeder, B) t.ok(seeder._groupServed.has(R), 'the root stays') - // B goes; a reader that dials the seeder only reads W from block 0 up. + // B goes; a reader that dials the seeder only gets the leave, verified. await B.swarm.destroy() const r = await reader(t, bootstrap, publicKey) const copy = r.store.get({ key: b4a.from(W, 'hex') }) await copy.ready() await until(() => copy.length === 8, { name: 'comprimento assinado no leitor', timeoutMs: 30000 }) - const types = [] - for (let i = 0; i < 8; i++) types.push(JSON.parse(b4a.toString(await copy.get(i, { timeout: 15000 }))).type) - t.is(types[6], 'group-leave', 'the leave of a released writer is reached by a contiguous read') + t.ok(await copy.get(0, { timeout: 15000 }), 'block 0 verified from the seeder') + t.is(JSON.parse(b4a.toString(await copy.get(6, { timeout: 15000 }))).type, 'group-leave', 'the leave of a released writer is served, verified') + await t.exception(copy.get(1, { timeout: 3000 }), 'a post is not served') t.is(events.filter((e) => e.event === 'fatal').length, 0) }) -test('a restart keeps a cut writer and a released one whole (#350)', async (t) => { +test('a restart restores a writer at its persisted cut and the scan lowers it or makes it a stub (#350)', async (t) => { t.timeout(240000) const { bootstrap } = await testnet(t) const assoc = makeAssociation() @@ -363,8 +383,10 @@ test('a restart keeps a cut writer and a released one whole (#350)', async (t) = await admitted(first.events, W2) await settledFor(first.seeder, B) // Held before the restart, or the restore drops them as headerless. - await downloaded(first.seeder, W, 6) + await until(() => first.seeder._groups.get(W).clearedTo === 6 && !first.seeder._groupServed.get(W).cutting, { name: 'W lido depois do corte' }) + t.alike(await blocksHeld(first.seeder._groupServed.get(W).core, 0, 6), [0, 1, 2, 3]) await downloaded(first.seeder, W2, 2) + t.is(readGroups(dataDir)[W].cut, 4) await first.seeder.close() // While the seeder is down B lowers W's cut and releases W2. @@ -372,41 +394,180 @@ test('a restart keeps a cut writer and a released one whole (#350)', async (t) = await B.diary.append(groupCutRecord({ group: R, writer: W2, cut: 0 })) const second = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex }) - t.ok(second.seeder._groupServed.get(W), 'W served at ready') - await settledFor(second.seeder, B) - t.alike(await blocksHeld(second.seeder._groupServed.get(W).core, 0, 6), [0, 1, 2, 3, 4, 5], 'W whole') - t.alike(await blocksHeld(second.seeder._groupServed.get(W2).core, 0, 2), [0, 1], 'W2 released, whole') + const entry = second.seeder._groupServed.get(W) + t.ok(entry, 'W served at ready') + t.ok(entry.cut === 4 || entry.cut === 2, 'from its persisted cut') + await until(async () => entry.cut === 2 && !entry.cutting && (await blocksHeld(entry.core, 0, 6)).length === 2, { name: 'W cortado em 2' }) + t.alike(await blocksHeld(entry.core, 0, 6), [0, 1]) + t.is(readGroups(dataDir)[W].cut, 2, 'the lower cut is persisted') + t.is(readGroups(dataDir)[W].clearedFrom, 2, 'and the range read reaches down to it') + const stub = second.seeder._groupServed.get(W2) + await until(async () => stub.cut === 0 && !stub.cutting && (await blocksHeld(stub.core, 0, 2)).length === 1, { name: 'W2 reduzido a um toco' }) + t.alike(await blocksHeld(stub.core, 0, 2), [0], 'W2 released: its block 0 stays') + await until(() => readGroups(dataDir)[W2].cut === 0, { name: 'toco persistido' }) t.absent(second.events.some((e) => e.event === 'group-rejected' && (e.key === W || e.key === W2)), 'never purged') t.is(second.events.filter((e) => e.event === 'fatal').length, 0) }) -test('a cut writer is held to its whole log\'s quota: refused past it at admission, dropped once it grows past it (#350)', async (t) => { +test('a cut that is gone after a restart fetches again what clearing removed (#350)', async (t) => { + t.timeout(240000) + const { bootstrap } = await testnet(t) + const assoc = makeAssociation() + const { dataDir } = prepareSeederDir(t, assoc) + const first = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex }) + + // B's diary: certificate, the join of W, then the cut at 3 (block 2). + const { A, B, R, W } = await cutWorld(t, bootstrap, assoc, (R, W) => [groupCutRecord({ group: R, writer: W, cut: 3 })]) + await admit(A, assoc) + await admit(B, assoc) + await admitted(first.events, W) + await until(async () => first.seeder._groups.get(W).clearedTo === 6 && !first.seeder._groupServed.get(W).cutting, { name: 'W lido depois do corte' }) + t.alike(await blocksHeld(first.seeder._groupServed.get(W).core, 0, 6), [0, 1, 2], 'cleared past its cut') + + // B deletes the cut record: the seeder clears that block, so after a + // restart its scan never sees the cut. + await B.diary.append(deleteRecord({ seq: 2 })) + const diaryCopy = first.seeder._served.get(B.key).core + await until(async () => !(await diaryCopy.has(2)), { name: 'corte apagado no semeador' }) + t.is(first.seeder._groups.get(W).cut, 3, 'the running seeder keeps the cut') + await first.seeder.close() + + const second = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex }) + const entry = second.seeder._groupServed.get(W) + t.ok(entry, 'W served at ready') + t.is(entry.cut, 3, 'from its persisted cut') + await until(async () => entry.cut === null && (await blocksHeld(entry.core, 0, 6)).length === 6, { name: 'W buscado de novo' }) + t.alike(await blocksHeld(entry.core, 0, 6), [0, 1, 2, 3, 4, 5], 'every block fetched again') + await until(() => !('cut' in readGroups(dataDir)[W]), { name: 'sem corte persistido' }) + t.alike(readGroups(dataDir)[W], { diary: B.key, group: R, seq: 1 }, 'the binding has no cut nor range') + t.absent(second.events.some((e) => e.event === 'group-rejected' && e.key === W), 'never purged') + t.is(second.events.filter((e) => e.event === 'fatal').length, 0) +}) + +test('a cut writer counts what it keeps toward the log quota: posts past the cut are cleared, not counted (#350)', async (t) => { t.timeout(150000) const { bootstrap } = await testnet(t) const assoc = makeAssociation() const { dataDir } = prepareSeederDir(t, assoc) const { seeder, events } = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex, maxLogBytes: 20000 }) - // A raw block near the block limit is enough to cross the byte quota. + // Blocks past a cut are only read at the seeder, never validated: a raw + // block near the block limit is enough to cross the byte quota. const big = () => b4a.from(JSON.stringify({ type: 'group-post', text: 'x'.repeat(12000) })) - // W is cut at 2 and already holds 24 KB past its cut when B is admitted. + const bigKey = () => b4a.from(JSON.stringify({ type: 'group-key', pad: 'x'.repeat(12000) })) + + // W is cut at 2 and already holds 24 KB of posts past its cut when B is + // admitted: they are cleared, so W is served. const { A, B, R, W } = await cutWorld(t, bootstrap, assoc, (R, W) => [groupCutRecord({ group: R, writer: W, cut: 2 })]) await B.cores.w.append(big()) await B.cores.w.append(big()) await admit(A, assoc) await admit(B, assoc) - t.is((await rejected(events, W, 'quota')).reason, 'quota', 'refused at admission: the bytes past the cut count') - t.absent(seeder._groupServed.has(W)) - - // W2, cut at 2 while small, is served; it then grows past the quota. + await admitted(events, W) + const entry = seeder._groupServed.get(W) + await until(async () => !entry.cutting && seeder._groups.get(W).clearedTo === 8 && (await blocksHeld(entry.core, 0, 8)).length === 2, { name: 'W lido depois do corte' }) + t.alike(await blocksHeld(entry.core, 0, 8), [0, 1], 'the posts past the cut are cleared') + t.absent(events.some((e) => e.event === 'group-rejected' && e.key === W), 'and not counted') + + // Two key blocks past the cut stay, and they are past the quota. + await B.cores.w.append(bigKey()) + await B.cores.w.append(bigKey()) + t.is((await rejected(events, W, 'quota')).reason, 'quota', 'dropped once what it keeps passes the quota') + await until(() => !seeder._groupServed.has(W), { name: 'W fora' }) + + // W2, cut at 4 while small, is served; it then grows past the quota + // below its cut. const W2 = await addWriter(B, R, 'w2') await B.diary.append(groupJoinRecord({ group: R, writer: W2, on: true })) - await B.diary.append(groupCutRecord({ group: R, writer: W2, cut: 2 })) + await B.diary.append(groupCutRecord({ group: R, writer: W2, cut: 4 })) await admitted(events, W2) await downloaded(seeder, W2, 2) - t.ok(seeder._groupServed.has(W2), 'served while within the quota') + await until(() => seeder._groupServed.get(W2).cut === 4, { name: 'W2 em 4' }) await B.cores.w2.append(big()) await B.cores.w2.append(big()) - t.is((await rejected(events, W2, 'quota')).reason, 'quota', 'dropped once its log passes the quota') + t.is((await rejected(events, W2, 'quota')).reason, 'quota', 'the bytes below the cut count') await until(() => !seeder._groupServed.has(W2), { name: 'W2 fora' }) + await settledFor(seeder, B) + t.is(events.filter((e) => e.event === 'group-admitted' && e.key === W).length, 1, 'a writer dropped for what it keeps is not fetched again') + t.is(events.filter((e) => e.event === 'fatal').length, 0) +}) + +test('a lower cut seen while a writer core is still opening is the cut it is served at (#350)', async (t) => { + t.timeout(150000) + const { bootstrap } = await testnet(t) + const assoc = makeAssociation() + const { dataDir } = prepareSeederDir(t, assoc) + const hold = gate() + const { seeder, events } = await startSeeder(t, bootstrap, dataDir, { + associationKey: assoc.hex, + admitHook: (key, step) => step === 'open' ? hold.hook(key) : undefined + }) + t.teardown(() => hold.open()) + + const { A, B, R, W } = await cutWorld(t, bootstrap, assoc, (R, W) => [groupCutRecord({ group: R, writer: W, cut: 4 })]) + hold.key = W + await admit(A, assoc) + await admit(B, assoc) + await until(() => hold.entered, { name: 'W abrindo' }) + t.is(seeder._groups.get(W).cut, 4, 'admitted at its cut') + await B.diary.append(groupCutRecord({ group: R, writer: W, cut: 2 })) + await until(() => seeder._groups.get(W).cut === 2, { name: 'corte menor na ligação', timeoutMs: 30000 }) + t.absent(seeder._groupServed.has(W), 'while the core is still opening') + await until(() => readGroups(dataDir)[W].cut === 2, { name: 'corte menor persistido' }) + hold.open() + await admitted(events, W) + const entry = seeder._groupServed.get(W) + t.is(entry.cut, 2, 'served at the lower cut') + await until(async () => !entry.cutting && seeder._groups.get(W).clearedTo === 6 && (await blocksHeld(entry.core, 0, 6)).length === 2, { name: 'W abaixo de 2' }) + t.alike(await blocksHeld(entry.core, 0, 6), [0, 1]) t.is(events.filter((e) => e.event === 'fatal').length, 0) }) + +test('a boxless writer of a private group its diary released is kept as a stub; before the release it is refused (#350)', async (t) => { + t.timeout(180000) + const { bootstrap } = await testnet(t) + const assoc = makeAssociation() + const { dataDir, publicKey } = prepareSeederDir(t, assoc) + const first = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex }) + + let P = null + const A = await member(t, bootstrap, assoc, async ({ key, make }) => { + P = await make('root', groupHeader({ diary: key, visibility: 'private', join: 'approval', name: null, box: 'e'.repeat(64) })) + return [groupJoinRecord({ group: P, writer: P, on: true })] + }) + // N: a writer of B in the private group without a box (one its diary + // replaced), header and a post. + let N = null + const B = await member(t, bootstrap, assoc, async ({ key, make }) => { + N = await make('n', writerHeader({ group: P, diary: key }), [groupPost({ text: 'oi' })]) + return [groupJoinRecord({ group: P, writer: N, on: true })] + }) + await admit(A, assoc) + await admitted(first.events, P) + await until(() => first.seeder._groupVisibility(P) === 'private', { name: 'raiz privada conhecida' }) + await admit(B, assoc) + await rejected(first.events, N, 'header-invalid') + t.absent(first.seeder._groupServed.has(N), 'refused while counted') + + await B.diary.append(groupCutRecord({ group: P, writer: N, cut: 0 })) + await admitted(first.events, N) + const entry = first.seeder._groupServed.get(N) + await until(async () => !entry.cutting && first.seeder._groups.get(N).clearedTo === 2 && (await blocksHeld(entry.core, 0, 2)).length === 1, { name: 'N reduzido a um toco' }) + t.alike(await blocksHeld(entry.core, 0, 2), [0], 'released: its block 0 is kept') + t.is(readGroups(dataDir)[N].cut, 0) + + // A reader fetches the stub's header from the seeder alone. + await B.swarm.destroy() + const r = await reader(t, bootstrap, publicKey) + const copy = r.store.get({ key: b4a.from(N, 'hex') }) + await copy.ready() + const header = JSON.parse(b4a.toString(await copy.get(0, { timeout: 20000 }))) + t.is(header.type, 'group-writer') + t.is(header.box, null, 'the boxless header readers reject') + await first.seeder.close() + + // A restart keeps the stub: its header is judged as a released writer's. + const second = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex }) + t.ok(second.seeder._groupServed.get(N), 'N served at ready') + t.absent(second.events.some((e) => e.event === 'group-rejected' && e.key === N), 'not purged') + t.is(first.events.filter((e) => e.event === 'fatal').length + second.events.filter((e) => e.event === 'fatal').length, 0) +}) From ef5df141c0f203249d6249cc2ce31a8a9be95227 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 01:24:15 -0300 Subject: [PATCH 06/25] test(core): fold sparse cut writers fed by a seeder like the whole copies A fresh member that syncs only with a seeder holds a frozen writer below its cut plus its leave, and a released writer as a stub, and folds the community and both groups exactly as after taking the whole copies from their owner (#350). --- packages/core/test/sync-groups-sparse.test.js | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 packages/core/test/sync-groups-sparse.test.js diff --git a/packages/core/test/sync-groups-sparse.test.js b/packages/core/test/sync-groups-sparse.test.js new file mode 100644 index 00000000..1666a3c3 --- /dev/null +++ b/packages/core/test/sync-groups-sparse.test.js @@ -0,0 +1,140 @@ +'use strict' + +// A seeder keeps a writer its diary cut sparse (#350, docs/protocol.md +// section 10): below the cut, then only its leaves and key blocks, and a +// released writer as a stub. A fresh member fed by that seeder alone folds +// the group as one holding the whole copies. +const test = require('brittle') +const b4a = require('b4a') +const { testnet } = require('./helpers/net') +const { generateAssociationKeyPair } = require('../lib/cert') +const { groupCutRecord } = require('../lib/records') +const { groupPost, groupLeave } = require('../lib/group') +const store = require('../lib/store') +const member = require('../lib/member') +const { person, exchange, startSeeder, syncOnce, ownWriter, runUntil } = require('./helpers/sync-groups') +const { offlineAppend, writerEntries } = require('./helpers/group-member') + +async function blocksHeld (core, from, to) { + const out = [] + for (let i = from; i < to; i++) if (await core.has(i)) out.push(i) + return out +} + +// The blocks of each writer `ctx` holds, by key. +async function heldCopies (ctx, keys) { + const s = await store.openStore(ctx.dataDir) + try { + const out = {} + for (const key of keys) { + const core = s.get({ key: b4a.from(key, 'hex'), writable: false }) + await core.ready() + out[key] = { length: core.length, contiguous: core.contiguousLength, held: await blocksHeld(core, 0, core.length) } + await core.close() + } + return out + } finally { + await s.close() + } +} + +async function view (ctx, community, groups) { + const out = { + community: { + info: await member.communityInfo(ctx, { community }), + members: await member.communityMembers(ctx, { community }), + groups: await member.communityGroups(ctx, { community }) + } + } + for (const group of groups) { + const info = await member.groupInfo(ctx, { group }) + const members = await member.groupMembers(ctx, { group }) + const page = await member.groupRead(ctx, { group }) + out[group] = { info, members, posts: page.items.map((p) => [p.writer, p.seq, p.text]) } + } + return out +} + +test('a fresh member fed only by a seeder folds a sparse frozen writer and a released stub like the whole copies (#350)', async (t) => { + t.timeout(400000) + const { bootstrap } = await testnet(t) + const kp = generateAssociationKeyPair() + const seeder = await startSeeder(t, bootstrap, kp) + const a = await person(t, kp, 'Ana') + const b = await person(t, kp, 'Bia') + const r = await person(t, kp, 'Rui') + const { community } = await member.communityCreate(a, { name: 'Bairro', join: 'open' }) + const { group: g1 } = await member.groupCreate(a, { name: 'Horta', join: 'open', community }) + const { group: g2 } = await member.groupCreate(a, { name: 'Feira', join: 'open', community }) + await member.groupPost(a, { group: g1, text: 'primeiro' }) + await member.groupPost(a, { group: g2, text: 'bem-vindos' }) + // B and R learn the roots out of band; everything else travels by sync. + await exchange(a, b) + await exchange(a, r) + for (const m of [b, r]) { + await member.communityJoin(m, { community }) + await member.groupJoin(m, { group: g1 }) + await member.groupJoin(m, { group: g2 }) + } + await member.groupPost(b, { group: g1, text: 'antes' }) + await member.groupPost(b, { group: g2, text: 'oi' }) + + // W1, B's writer in G1, is frozen at its length: two posts and a leave + // past the cut. W2, its writer in G2, is released: a post and a leave. + const w1 = ownWriter(b, g1) + const w2 = ownWriter(b, g2) + const cut1 = (await writerEntries(b, g1)).length + await offlineAppend(b, g1, groupPost({ text: 'depois' })) + await offlineAppend(b, g1, groupPost({ text: 'depois 2' })) + const leave1 = await offlineAppend(b, g1, groupLeave({ seen: null, cuts: [{ writer: w1, seq: cut1 + 2 }] })) + const len2 = (await writerEntries(b, g2)).length + await offlineAppend(b, g2, groupPost({ text: 'tchau' })) + const leave2 = await offlineAppend(b, g2, groupLeave({ seen: null, cuts: [{ writer: w2, seq: len2 + 1 }] })) + const s = await store.openStore(b.dataDir) + try { + const diary = await store.openDiary(s) + await store.appendRecord(diary, groupCutRecord({ group: g1, writer: w1, cut: cut1 })) + await store.appendRecord(diary, groupCutRecord({ group: g2, writer: w2, cut: 0 })) + } finally { + await s.close() + } + + const ra = await syncOnce(bootstrap, a) + t.ok(ra.result.ok, 'A pushes its diary and roots: ' + ra.result.errors.join('; ')) + // B's run pushes its diary and writers; it stops once the seeder holds + // both writers sparse. + const sparse = async () => { + const e1 = seeder._groupServed.get(w1) + const e2 = seeder._groupServed.get(w2) + const b1 = seeder._groups.get(w1) + const b2 = seeder._groups.get(w2) + if (!e1 || !e2 || !b1 || !b2 || e1.cutting || e2.cutting) return false + if (b1.cut !== cut1 || b1.clearedTo !== leave1.seq + 1 || b2.cut !== 0 || b2.clearedTo !== leave2.seq + 1) return false + const h1 = await blocksHeld(e1.core, 0, leave1.seq + 1) + const h2 = await blocksHeld(e2.core, 0, leave2.seq + 1) + return h1.length === cut1 + 1 && h2.length === 2 + } + await runUntil(b, bootstrap, sparse) + t.alike(await blocksHeld(seeder._groupServed.get(w1).core, 0, leave1.seq + 1), [...Array(cut1).keys(), leave1.seq], 'the seeder keeps W1 below its cut and its leave') + t.alike(await blocksHeld(seeder._groupServed.get(w2).core, 0, leave2.seq + 1), [0, leave2.seq], 'and W2 as a stub') + + // R, fresh, syncs with the seeder alone (B is offline). + await syncOnce(bootstrap, r, { timeoutMs: 30000 }) + const copies = await heldCopies(r, [w1, w2]) + t.alike(copies[w1], { length: leave1.seq + 1, contiguous: cut1, held: [...Array(cut1).keys(), leave1.seq] }, 'R holds W1 sparse') + t.alike(copies[w2], { length: leave2.seq + 1, contiguous: 1, held: [0, leave2.seq] }, 'and W2 as a stub') + const fromSeeder = await view(r, community, [g1, g2]) + + // The whole copies, straight from B. + await exchange(b, r) + const whole = await heldCopies(r, [w1, w2]) + t.is(whole[w1].contiguous, leave1.seq + 1, 'control: R now holds W1 whole') + t.is(whole[w2].contiguous, leave2.seq + 1, 'and W2') + const full = await view(r, community, [g1, g2]) + const texts = (g) => full[g].posts.map((p) => p[2]) + t.ok(texts(g1).includes('antes'), 'control: the post below the cut counts') + t.absent(texts(g1).includes('depois'), 'control: a post past the cut does not') + t.absent(texts(g2).includes('oi'), 'control: a released writer counts no post') + t.absent(full[g1].members.members.some((m) => m.member === b.key), 'control: the leave past the cut counts') + t.alike(fromSeeder, full, 'the sparse copies fold like the whole ones') +}) From d946028b424f0688ef9b55c745cc5cf1d343ebd6 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 01:29:35 -0300 Subject: [PATCH 07/25] docs: describe sparse cut writers on seeders and readers The protocol states what a seeder keeps of a cut or released writer, how readers read a writer past its first hole and how the fold counts it, and that a key block citing a cut CW is judged by the signed length (#350). Adds the changelog fragment and the verification file of the branch. --- docs/changelog/feat-writer-slot-sparse.md | 3 + docs/protocol.md | 18 +-- docs/verification/feat-writer-slot-sparse.md | 110 +++++++++++++++++++ 3 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 docs/changelog/feat-writer-slot-sparse.md create mode 100644 docs/verification/feat-writer-slot-sparse.md diff --git a/docs/changelog/feat-writer-slot-sparse.md b/docs/changelog/feat-writer-slot-sparse.md new file mode 100644 index 00000000..19f3826b --- /dev/null +++ b/docs/changelog/feat-writer-slot-sparse.md @@ -0,0 +1,3 @@ +### Alterado + +- Os semeadores passam a guardar só o que conta de um escritor cortado pelo diário (PR `feat/writer-slot-sparse`, parte da #350): os blocos abaixo do corte e, a partir dele, só as saídas do grupo e os blocos de chave. De um escritor liberado (corte 0) fica um toco: o bloco 0, as saídas e os blocos de chave. A cota conta o que fica guardado. O nó core, o nó do navegador e o serviço de contas leem um escritor além do primeiro buraco (só as saídas e os blocos de chave), então quem só fala com semeadores vê o grupo como quem tem as cópias inteiras. Um bloco de chave que cita um escritor de comunidade cortado vale pelo comprimento que o escritor assinou. O escritor sem caixa trocado num grupo privado, depois de liberado, também fica como toco no semeador. Conferência em `docs/verification/feat-writer-slot-sparse.md`. diff --git a/docs/protocol.md b/docs/protocol.md index dd9c5f79..1b581d4b 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -999,7 +999,7 @@ A seeder `hello` additionally carries `"cert": { "v": 1, "seeder": "", "d - The admission channel's open handshake carries one compact-encoding `uint`, the sender's capability bits; with no bits set (caps 0) the open carries no bytes at all, so a peer without capabilities opens the channel byte for byte as before. Decoding is tolerant: an open with no bytes (every peer from before capabilities) or with bytes that do not parse reads as caps 0, and unknown bits are ignored, so either side may be older than the other. `attachAdmission` takes the option `caps` (a non-negative safe integer, else `TypeError`, default 0) and exposes the peer's bits as `Peer.remoteCaps`. - `CAP_NO_INVENTORY = 1`: sent by a member that wants no inventory from the seeder (the lean hello of "Bounded inventory" above). -- `CAP_WRITER_CUTS = 4`: sent by every seeder that keeps the writers a diary cut (section 10, "Seeder group admission"): per diary and group up to 8 live and 8 frozen writers chosen by the slot walk and up to 16 writers the diary released, each held whole under the whole-log quota. Members may use it to tell such a seeder from an older one, which counts at most 8 writers per diary and group by earliest join and never frees a slot. This version only announces it. +- `CAP_WRITER_CUTS = 4`: sent by every seeder that keeps the writers a diary cut (section 10, "Seeder group admission"): per diary and group up to 8 live and 8 frozen writers chosen by the slot walk and up to 16 writers the diary released, a frozen one kept below its cut plus its leaves and key blocks and a released one as a stub (block 0, leaves and key blocks), under the log quota on what is kept. Members may use it to tell such a seeder from an older one, which counts at most 8 writers per diary and group by earliest join and never frees a slot. This version only announces it. - `CAP_DISCOVERY = 2`: sent by a seeder that answers `'varanda/discovery/1'` and sends lean hellos. The seeder sends it unless it was started with `discovery: false` (constructor option, default true, no CLI flag: tests and rollback), in which case it behaves exactly like a seeder from before discovery: no bit, no lean hello, no discovery channel (a member's discovery open is rejected). - The capabilities change nothing else: the message shapes and limits of this section, `maxServed`, `maxExternal`, the frame limits and the hello of a peer that sends no bit are unchanged. @@ -1882,13 +1882,13 @@ A CW record (admit, ban, hide or invites) whose `group` names a key outside C's `MAX_GROUP_WRITERS_PER_DIARY` = 8: `buildGroup` counts per diary the bound writers its slot walk counts (below): at most 8 live ones, a bound root one of its creator's 8, plus at most `MAX_FROZEN_WRITERS_PER_DIARY` = 8 frozen ones. `MAX_GROUP_WRITERS` = 512: of the writers that remain, the fold keeps the root, then the other writers in this order of tiers: those of the creator's diary (root bound only), those of the diaries that are admins in the final root state (group admins, and the creator and admins of the group's community), all others, and last those of banned diaries (ranked by the final ban layer); inside a tier the live writers before the frozen ones, then by key. A rejoining writer whose named leave fell outside the cap is dropped as well, because the fold would otherwise never see that leave. `incomplete` is true when the 512 cap or that rejoin rule drops a writer; the per-diary cap alone does not set it. `MAX_GROUPS_PER_DIARY` = 64 is enforced per diary by the seeder, which keeps the 64 EARLIEST live joins by diary seq (the root counts toward the 512 writer cores of a group at the seeder). `MAX_GROUPS_PER_COMMUNITY` = 32 counts the `group-create` records of a community in its causal order ("Communities contain groups"). `MAX_GROUP_CUTS` = 16 caps the cuts of admits, bans and admins, and also of a community `group-admin` (over the member's CWs, of which a diary binds at most 8): there is no separate community cut cap. A community core set (the root C and its CWs) uses the same per-diary and total caps and the same tiers as a group. `MAX_GROUP_NAME` = 80. Blocks use `MAX_BLOCK_BYTES` = 16384; each writer core uses section 2's `MAX_LOG_BLOCKS` = 100000 and `MAX_LOG_BYTES` = 64 MiB. Text length is defined in the post shape above. -**Slot walk** (#350, `slotWalk` in `group.js`). Per diary d and root G (or community C), the counted writers come from d's own `group-join` and `group-cut` records for G in diary seq order and nothing else, so every reader and every seeder agrees, and a prefix of the diary never disagrees with the whole diary about the records it holds. A writer's state only moves forward: none, live, frozen, released. (1) An `on: true` join of a writer in state none makes it live while fewer than 8 are live (7 from d's first `on: true` join of the root key G itself: the root slot, which any diary joining writer G reserves from that join on); with no room the writer is over and never counts, even when a slot frees later. A repeated or `on: false` join changes no slot. (2) A mark on a live or frozen writer lowers its cut (the lowest wins, and it always applies to the writer's records, "Diary cuts on own writers"). Cut 0 releases the writer while d released fewer than `MAX_WRITER_RELEASES_PER_DIARY` = 16 writers in G; a cut >= 1 freezes a live writer while fewer than 8 are frozen; with no room the writer keeps its state and its slot, with the cut. (3) A mark on a writer in state none or released, every mark on the root, and a mark with no valid seq are ignored. The candidates are the live, frozen and released writers. A writer cut at 0 (released, or kept at cut 0 once the 16 releases are spent) gives its slot back when there is room, but its block 0, its leaves and its key blocks still count ("Diary cuts on own writers"), so it is awaited like a frozen writer, and a released one ranks after the frozen writers of its tier under the 512 cap. A released writer is awaited whatever its diary's newest join says, in a private group too, so a diary that turns off and cuts at 0 the writer holding its leave never gets a fold that looks complete without that leave. Once held, a writer cut at 0 whose header box does not match the group's kind (the boxless writer a node replaced, "Writers named by a node") is rejected, never awaited; seeders and sync refuse such a header with `header-invalid`, so a reader that never held it keeps the fold incomplete. A writer that is over is no candidate, is never awaited and never makes the fold incomplete. Releases bound the churn: a loop of join, cut 0, join stops after 16 releases and 8 writers held at cut 0, so a diary has at most 8 live, 8 frozen and 16 released writers in G. A rejoin naming a leave on a writer of the same diary that its walk voided (over) is rejected; one naming a leave of a released writer is valid, since that leave still counts. A hole in the history of a diary with an `on: true` join for G (`{ gap: true }` in its marks) may hide the mark that freed or froze a slot, so it sets `incomplete`. The fold returns `frozen` (the frozen writers, released ones included), `released` (the released writers it kept) and `slots` (diary -> `{ live, frozen, released }`, the counts of each diary with an `on: true` join for G, from which nodes derive the room left). `diaryGroupKeys` and the seeder ("Seeder group admission") use the same walk, so a second device of the account and the seeders open the same roots and writers. +**Slot walk** (#350, `slotWalk` in `group.js`). Per diary d and root G (or community C), the counted writers come from d's own `group-join` and `group-cut` records for G in diary seq order and nothing else, so every reader and every seeder agrees, and a prefix of the diary never disagrees with the whole diary about the records it holds. A writer's state only moves forward: none, live, frozen, released. (1) An `on: true` join of a writer in state none makes it live while fewer than 8 are live (7 from d's first `on: true` join of the root key G itself: the root slot, which any diary joining writer G reserves from that join on); with no room the writer is over and never counts, even when a slot frees later. A repeated or `on: false` join changes no slot. (2) A mark on a live or frozen writer lowers its cut (the lowest wins, and it always applies to the writer's records, "Diary cuts on own writers"). Cut 0 releases the writer while d released fewer than `MAX_WRITER_RELEASES_PER_DIARY` = 16 writers in G; a cut >= 1 freezes a live writer while fewer than 8 are frozen; with no room the writer keeps its state and its slot, with the cut. (3) A mark on a writer in state none or released, every mark on the root, and a mark with no valid seq are ignored. The candidates are the live, frozen and released writers. A writer cut at 0 (released, or kept at cut 0 once the 16 releases are spent) gives its slot back when there is room, but its block 0, its leaves and its key blocks still count ("Diary cuts on own writers"), so it is awaited like a frozen writer, and a released one ranks after the frozen writers of its tier under the 512 cap. A released writer is awaited whatever its diary's newest join says, in a private group too, so a diary that turns off and cuts at 0 the writer holding its leave never gets a fold that looks complete without that leave. Once held, a writer cut at 0 whose header box does not match the group's kind (the boxless writer a node replaced, "Writers named by a node") is rejected, never awaited; the member sync refuses such a header with `header-invalid` before storing it, so a reader that never held it keeps the fold incomplete; seeders refuse it while it is counted and keep it as a stub once released, so its block 0 stays available ("Seeder group admission"). A writer that is over is no candidate, is never awaited and never makes the fold incomplete. Releases bound the churn: a loop of join, cut 0, join stops after 16 releases and 8 writers held at cut 0, so a diary has at most 8 live, 8 frozen and 16 released writers in G. A rejoin naming a leave on a writer of the same diary that its walk voided (over) is rejected; one naming a leave of a released writer is valid, since that leave still counts. A hole in the history of a diary with an `on: true` join for G (`{ gap: true }` in its marks) may hide the mark that freed or froze a slot, so it sets `incomplete`. The fold returns `frozen` (the frozen writers, released ones included), `released` (the released writers it kept) and `slots` (diary -> `{ live, frozen, released }`, the counts of each diary with an `on: true` join for G, from which nodes derive the room left). `diaryGroupKeys` and the seeder ("Seeder group admission") use the same walk, so a second device of the account and the seeders open the same roots and writers. -**Writers named by a node** (`countedCuts` and `groupCuts` in `member.js`). In a community, an admit, a ban or a demotion that a node writes names every counted writer of the member, a frozen one at its cut, never past it, and never a released one (8 live and 8 frozen fit `MAX_GROUP_CUTS`; 16 released writers would not, and only their block 0, leaves and key blocks count). In a group, an admit, a ban or a demotion names the writers a new post would count on (a probe post past each writer's length): a frozen writer whose cut is at or below its length takes no new post and is left out, its cut already fixes it, and one cut past its length is named, never past the diary's cut. A leave and an `on: true` grant name the live writers only, so they stay within 8 cuts. The writers an admission waits for (`unheldCountedWriters`) come from the walk too: a frozen or released writer is awaited (its leaves still count), except, in a private group, one whose newest join is off: an admit never names a released writer, and a replaced boxless one can never be held (seeders and sync refuse it), so waiting for it would hold the admission for good. The fold itself still awaits it ("Limits"). A node that replaces its own writer of a private group whose header box is not the device's (a join before the root arrived; `joinLoaded` in the core and browser nodes) writes, after that writer's join off and before the new writer's join, a `group-cut` on it (`replacedWriterCut` in `group.js`), and never a second one at the same or a higher cut. The cut is 0 for a header with no box: the walk releases it whatever it holds, and nothing past its block 0, leaves and key blocks ever counted in a private group, where a boxless writer is rejected. For a header with a box it is one past the last seq the node holds (1 for a header with a rejoin and nothing else, 0 for a bare header), so nothing it holds stops counting. The join off alone keeps its slot in the walk. With an unknown length (the core not held) no cut is written. Before writing the join off, the cut or the new join, the node steps its diary's own walk with those events (`walkHasRoom` in `group.js`): when the new writer would not take a live slot, it writes nothing and fails with `GROUP_FULL`. The other automatic cuts (a writer replaced after a leave or a retired CW) and the cut of a disconnected device come with the client nodes. +**Writers named by a node** (`countedCuts` and `groupCuts` in `member.js`). In a community, an admit, a ban or a demotion that a node writes names every counted writer of the member, a frozen one at its cut, never past it, and never a released one (8 live and 8 frozen fit `MAX_GROUP_CUTS`; 16 released writers would not, and only their block 0, leaves and key blocks count). In a group, an admit, a ban or a demotion names the writers a new post would count on (a probe post past each writer's length): a frozen writer whose cut is at or below its length takes no new post and is left out, its cut already fixes it, and one cut past its length is named, never past the diary's cut. A leave and an `on: true` grant name the live writers only, so they stay within 8 cuts. The writers an admission waits for (`unheldCountedWriters`) come from the walk too: a frozen or released writer is awaited (its leaves still count), except, in a private group, one whose newest join is off: an admit never names a released writer, and a replaced boxless one is never held by the member sync, which refuses its header (seeders keep it only as a released stub), so waiting for it would hold the admission for good. The fold itself still awaits it ("Limits"). A node that replaces its own writer of a private group whose header box is not the device's (a join before the root arrived; `joinLoaded` in the core and browser nodes) writes, after that writer's join off and before the new writer's join, a `group-cut` on it (`replacedWriterCut` in `group.js`), and never a second one at the same or a higher cut. The cut is 0 for a header with no box: the walk releases it whatever it holds, and nothing past its block 0, leaves and key blocks ever counted in a private group, where a boxless writer is rejected. For a header with a box it is one past the last seq the node holds (1 for a header with a rejoin and nothing else, 0 for a bare header), so nothing it holds stops counting. The join off alone keeps its slot in the walk. With an unknown length (the core not held) no cut is written. Before writing the join off, the cut or the new join, the node steps its diary's own walk with those events (`walkHasRoom` in `group.js`): when the new writer would not take a live slot, it writes nothing and fails with `GROUP_FULL`. The other automatic cuts (a writer replaced after a leave or a retired CW) and the cut of a disconnected device come with the client nodes. ### Folding: `buildGroup` -`buildGroup({ root: { key, entries }, writers: [{ key, entries, diary }], joins: Map, community, keys, marks, confirms, now })` is a pure fold (`marks`, optional, is `Map` from `groupCutHistory`, "Diary cuts on own writers" and "Creator marks on group roots"; `buildCommunity` takes `marks` too; `confirms`, optional, is `Map` from `groupConfirmHistory` (`dseq` the confirmation's own seq in the diary), "Confirmed leaves"; `buildCommunity` takes `confirms` too); `community` (`null`, or the parent's input `{ root: { key, entries }, writers, joins }`, exactly `buildCommunity`'s input) is defined under "Communities contain groups", and `keys` (optional, `{ writers, unseal, onKey? }`: the reader's means to open epoch keys, and the hook that hands a node the accepted keys) and the private-only results `sealed`, `epoch`, `rekeyPending`, `entitled` and `removals` under "Private groups". It never throws and gives the same result for any delivery order of the same blocks. The first valid copy of a seq is retained. It returns `kind`, `name`, `join`, `visibility`, `creator`, `admins: Set` (the community's creator and admins included, below), `community` (the parent's fold result when C authorizes G, else null), `orphan` (true when C folds but does not authorize G, below), `members: Map`, `posts: [{ writer, diary, seq, at, text, replyTo, hidden }]` in causal order, `requests` (requested diaries), `future` (skipped post refs), `last` (the newest node in causal order, a rejoining writer's block 0 included, never a reaction, as `{ writer, seq }`, or null), `bans: Map` (the latest valid ban of every diary banned now: the `lifts` an unban must name), `leaves: Map` (the causal latest leave of every diary that has left now: the `rejoin` a new writer must name), `lastLeave: Map` (the causally latest leave of every diary that ever left, kept after a rejoin, as in a community), `liveWriters: Set` (the counted writers that are live now: not retired, not banned, not cut by their own diary, and, under the gate, with a live CW in C; the root only when `rootLive`), `authorityWriters: Set` (the live writers that hold moderation power now, for nodes to pick a writer that can sign an admin record), `cuts: Map` (each counted writer its own diary cut, with the lowest cut, "Diary cuts on own writers"; `buildCommunity` returns it for CWs), `cutUnknown: Set` (the diaries binding a counted writer whose marks as read have a hole, same section; `buildCommunity` returns it too), `writers: Set` (the writer cores the fold counted; the root only when bound), `memberInvite` (whether members may invite, "Members may invite") and `incomplete`. A missing named writer sets `incomplete: true` without blocking other writers. An invalid or absent root header yields the empty incomplete result, and so does a root whose header kind is `community` (a community is folded by `buildCommunity`, never by `buildGroup`). An unbound root (no `on: true` join for `writer = G` in the creator's diary) sets incomplete, carries no authority and contributes no posts: only its header is read, and the header's diary is an ordinary diary (not listed in `admins` or `members` unless it has counted writers of its own); `creator` in the result still echoes the header. A node whose own writer is outside `writers` (beyond the cap) refuses to write (`limite do grupo atingido`) and never reports `member` for it. +`buildGroup({ root: { key, entries }, writers: [{ key, entries, diary, signed? }], joins: Map, community, keys, marks, confirms, now })` is a pure fold (`marks`, optional, is `Map` from `groupCutHistory`, "Diary cuts on own writers" and "Creator marks on group roots"; `buildCommunity` takes `marks` too; `confirms`, optional, is `Map` from `groupConfirmHistory` (`dseq` the confirmation's own seq in the diary), "Confirmed leaves"; `buildCommunity` takes `confirms` too); `community` (`null`, or the parent's input `{ root: { key, entries }, writers, joins }`, exactly `buildCommunity`'s input) is defined under "Communities contain groups", and `keys` (optional, `{ writers, unseal, onKey? }`: the reader's means to open epoch keys, and the hook that hands a node the accepted keys) and the private-only results `sealed`, `epoch`, `rekeyPending`, `entitled` and `removals` under "Private groups". It never throws and gives the same result for any delivery order of the same blocks. The first valid copy of a seq is retained. It returns `kind`, `name`, `join`, `visibility`, `creator`, `admins: Set` (the community's creator and admins included, below), `community` (the parent's fold result when C authorizes G, else null), `orphan` (true when C folds but does not authorize G, below), `members: Map`, `posts: [{ writer, diary, seq, at, text, replyTo, hidden }]` in causal order, `requests` (requested diaries), `future` (skipped post refs), `last` (the newest node in causal order, a rejoining writer's block 0 included, never a reaction, as `{ writer, seq }`, or null), `bans: Map` (the latest valid ban of every diary banned now: the `lifts` an unban must name), `leaves: Map` (the causal latest leave of every diary that has left now: the `rejoin` a new writer must name), `lastLeave: Map` (the causally latest leave of every diary that ever left, kept after a rejoin, as in a community), `liveWriters: Set` (the counted writers that are live now: not retired, not banned, not cut by their own diary, and, under the gate, with a live CW in C; the root only when `rootLive`), `authorityWriters: Set` (the live writers that hold moderation power now, for nodes to pick a writer that can sign an admin record), `cuts: Map` (each counted writer its own diary cut, with the lowest cut, "Diary cuts on own writers"; `buildCommunity` returns it for CWs), `cutUnknown: Set` (the diaries binding a counted writer whose marks as read have a hole, same section; `buildCommunity` returns it too), `writers: Set` (the writer cores the fold counted; the root only when bound), `memberInvite` (whether members may invite, "Members may invite") and `incomplete`. A missing named writer sets `incomplete: true` without blocking other writers. An invalid or absent root header yields the empty incomplete result, and so does a root whose header kind is `community` (a community is folded by `buildCommunity`, never by `buildGroup`). An unbound root (no `on: true` join for `writer = G` in the creator's diary) sets incomplete, carries no authority and contributes no posts: only its header is read, and the header's diary is an ordinary diary (not listed in `admins` or `members` unless it has counted writers of its own); `creator` in the result still echoes the header. A node whose own writer is outside `writers` (beyond the cap) refuses to write (`limite do grupo atingido`) and never reports `member` for it. **Authority and layers.** @@ -1919,7 +1919,9 @@ A CW record (admit, ban, hide or invites) whose `group` names a key outside C's **Diary cuts on own writers** (owner decision, #350). A diary can cut any writer of its own, for instance the writer of a lost device: nothing that writer wrote at or past the cut counts, except its leaves and key blocks (below), and a cut is final ("Corte é final", owner decision): no record of another diary brings a dropped record back, whatever position it names (a demotion, ban or admit of d listing w at any length, or a basis ref on w). For a counted writer w (never G's root, never a community root) bound by diary d, `cut(w)` is the lowest `cut` among the valid `group-cut` marks of d for this group (or community) whose `writer` is w. Only d's own marks count: a mark in another diary naming w, or a mark of d naming a writer d does not bind, does nothing. -Binding drops the records of w at seq >= cut(w), as if w's core ended there, with three exceptions that always stay, whatever the cut: block 0 (the header and a rejoining writer's rejoin node), every `group-leave` (such a cut never undoes a leave, never revives a diary that left and never breaks a rejoin naming that leave, so `removals` and `leaves` are unchanged), and every `group-key` block. A leave written after the mark counts like one written before it (owner decision, 16/11/2026): a leave only takes, never gives, and a leave made on a lost device shortly before it was lost is not undone; the accepted limit is that a stolen device already offline can still take the person out of a group or an admin role, and the person asks to join again. A cut at 0 is the same rule with nothing kept below the cut (owner decision, 16/11/2026): w keeps block 0, every leave and every key block, stays a candidate and is awaited ("Limits", slot walk), and a rejoin naming one of its leaves stays valid. This holds for every mark, not only for honest nodes: whoever can sign d's diary (a thief with the unlocked identity included) can cut the owner's other device at 0 and free its slot, but never undo its leave. The records stop at the lowest cut among every valid mark of d on w the reader holds, even one the slot walk ignored for the slot (written before the join it reads, or held by a reader whose hole hides that join): the walk decides the slot, never the records. A key block past the cut stays valid for reading only: its keys open history as before, but it never sends and adds no recipient to rule 4 of "Eligibility". Everything else past the cut (posts, reactions, topics, polls, choices, events, hides, admits, bans, admin grants, meta, invite flags, sealed envelopes) does not count, and a reference into it is ignored like any reference past a core's end. Every later rule (order, authority, `lastSeq`, keys) sees the shortened log, except that a G key block whose `auth` cites a cut CW is judged against the CW's held length, so the history it handed out stays readable; it still gives no sending authority, since a cut CW is not in C's `authorityWriters`. A cut writer is in neither `liveWriters` nor `authorityWriters`. Its slot follows the slot walk ("Limits"): a cut >= 1 freezes it (still counted, after the live writers of its tier under the 512 cap), a cut 0 releases it, each while there is room; with no room it keeps its slot, cut. The fold returns `cuts` (writer -> cut). The result is the same for any order or arrival of the marks, before or after a rotation. +Binding drops the records of w at seq >= cut(w), as if w's core ended there, with three exceptions that always stay, whatever the cut: block 0 (the header and a rejoining writer's rejoin node), every `group-leave` (such a cut never undoes a leave, never revives a diary that left and never breaks a rejoin naming that leave, so `removals` and `leaves` are unchanged), and every `group-key` block. A leave written after the mark counts like one written before it (owner decision, 16/11/2026): a leave only takes, never gives, and a leave made on a lost device shortly before it was lost is not undone; the accepted limit is that a stolen device already offline can still take the person out of a group or an admin role, and the person asks to join again. A cut at 0 is the same rule with nothing kept below the cut (owner decision, 16/11/2026): w keeps block 0, every leave and every key block, stays a candidate and is awaited ("Limits", slot walk), and a rejoin naming one of its leaves stays valid. This holds for every mark, not only for honest nodes: whoever can sign d's diary (a thief with the unlocked identity included) can cut the owner's other device at 0 and free its slot, but never undo its leave. The records stop at the lowest cut among every valid mark of d on w the reader holds, even one the slot walk ignored for the slot (written before the join it reads, or held by a reader whose hole hides that join): the walk decides the slot, never the records. A key block past the cut stays valid for reading only: its keys open history as before, but it never sends and adds no recipient to rule 4 of "Eligibility". Everything else past the cut (posts, reactions, topics, polls, choices, events, hides, admits, bans, admin grants, meta, invite flags, sealed envelopes) does not count, and a reference into it is ignored like any reference past a core's end. Every later rule (order, authority, `lastSeq`, keys) sees the shortened log, except that a G key block whose `auth` cites a cut CW is judged by `auth.seq` against the CW's signed length ("Sparse copies", below), so the history it handed out stays readable, from a copy trimmed at the cut too; it still gives no sending authority, since a cut CW is not in C's `authorityWriters`. A cut writer is in neither `liveWriters` nor `authorityWriters`. Its slot follows the slot walk ("Limits"): a cut >= 1 freezes it (still counted, after the live writers of its tier under the 512 cap), a cut 0 releases it, each while there is room; with no room it keeps its slot, cut. The fold returns `cuts` (writer -> cut). The result is the same for any order or arrival of the marks, before or after a rotation. + +**Sparse copies** (#350). A seeder keeps a cut writer sparse ("Seeder group admission"): its blocks below the cut, then only its leaves and key blocks, so a reader may hold a writer up to a hole and some blocks past it. Every reader (the core node, the browser node and the account service) reads a writer up to its first missing block and, past it, only the `group-leave` and `group-key` blocks it holds, up to `MAX_LOG_BLOCKS` (`readHeldBlocks` in `store.js` and `node-store.js`), and hands the fold each copy's `signed`: the core length its writer signed, against which Hypercore verified every block held. Roots are still read as a prefix. The fold takes, of a writer, the records below the first seq no copy holds and, past that hole, only its leaves and key blocks, and only when the hole is at or past the writer's cut (`max(cut, 1)`, so a released writer needs its block 0): a sparse copy of a cut writer folds exactly like the whole copy, and a partial copy, cut or not, folds as the prefix it always was. A block past a hole below the cut never counts, since the whole copy would count the records missing before it. A cut writer's signed length is the largest `signed` among its copies; a key block whose `auth` cites a cut CW is valid for reading when `auth.seq` is at most that length (or the CW's held length, when greater), so a CW trimmed at its cut reads the history the whole copy reads. The browser node's fold memo stamps the blocks held past the contiguous length, so a block that arrives past a hole folds again. A `{ gap: true }` entry cuts nothing (the root rule under "Creator marks on group roots" keeps its own gap handling): a hole only delays a mark until it fills, and treating it as a cut would let any hole silence every device of the diary. The fold reports such diaries in `cutUnknown` (only those binding a counted writer), and the slot walk's gap rule ("Limits") sets `incomplete`. A node must not take the fold as complete about cuts for them: it keeps fetching that diary before deciding anything that depends on a writer of it being uncut, for instance before rotating away from it or treating its device as current. @@ -1949,7 +1951,7 @@ Late blocks can change the causal order of posts already displayed. The RPC surf A community C is the parent of groups. C is the key of a single-writer root core whose block 0 is a `group-header` with `kind: 'community'` (`community`, `member` and `box` null, `visibility` `public`, "Cores and headers"; `diary` is the community creator). Unlike a group root, C has writer cores: each device of a member writes a **community writer** (CW), a `group-writer` core with `group` = C and `member: null`, bound to its diary exactly like any writer (the join `{ group: C, writer: CW, on: true }` and a `rejoin` that names a `group-leave` of the same diary inside C). A group G has exactly one parent, the `community` of its header, immutable. C lists its groups in a catalogue made of `group-create` records ("Community-core records"); the first group of a community, created with it, is called Geral. Readers decide the kind from the root's own header. - **Bound root.** As for a group root, the header's `diary` is only a label until that diary binds the root with a join `{ group: C, writer: C, on: true }`. An unbound community root keeps only its header: it has no creator, no admins, no members and no catalogue, gives no power in any group, and `buildCommunity` returns `incomplete: true`. -- **`buildCommunity`.** `buildCommunity({ root: { key: C, entries }, writers: [{ key, entries, diary }], joins: Map, now })` is the input of `buildGroup` without `community` and `keys`. It runs the group fold on C's logs in community mode: the same binding (`readJoins`), per-diary cap, 512 cap with the same tiers, rejoin check, causal order, leave withdrawal, root admin layer (its cuts are over CW keys), admit and ban layers, shield and results `members`, `requests`, `bans`, `leaves`, `last` and `writers`. The differences are: +- **`buildCommunity`.** `buildCommunity({ root: { key: C, entries }, writers: [{ key, entries, diary, signed? }], joins: Map, now })` is the input of `buildGroup` without `community` and `keys`. It runs the group fold on C's logs in community mode: the same binding (`readJoins`), per-diary cap, 512 cap with the same tiers, rejoin check, causal order, leave withdrawal, root admin layer (its cuts are over CW keys), admit and ban layers, shield and results `members`, `requests`, `bans`, `leaves`, `last` and `writers`. The differences are: - Only records with `group: null` act in C. Records with `group !== null` (admit, ban, hide) are nodes in C's causal order and change nothing in C; a `group-hide` never sets a hide in C. - **Members may invite in C.** C's `memberInvite` is the `on` of the last `group-invites` record with `group: null` in C's causal order that counts as an administrative node of C: on C's root, the creator's while the root has its powers (root seq below `rootCut`); on a CW, the creator's diary or a diary whose C admin layer is on and C ban layer off at that CW and seq, the CW counted, not retired (no authority after leaving, past records included) and the record not withdrawn by a leave (`authorizedC` in `lib/group.js`, the gate of a `group: null` admit). A returning creator's record on a creator CW also counts when it passes the repair gate ("Community repair after the creator's return"), as for a CW meta. A member's record never counts. Absent, it is `false` (also for an unbound or empty C). A `group-invites` naming a group changes nothing in C. - A community has no posts: `posts: []`, `future: []`, `sealed: 0`, `epoch: null`, `rekeyPending: false`, `entitled: []`, `removals: []`. `join` and `name` come from the header and the last not-withdrawn `group-meta` by root seq, overridden by the latest counted CW meta ("Community repair after the creator's return"). @@ -2047,7 +2049,7 @@ A private group (`visibility: 'private'` in the root header, kind `group`) is a **Distributors.** A `group-key` block counts for reading when its writer was authorized to distribute at that writer/seq: a bound writer of the creator (root bound), or a writer that passes the ordinary authority check for administrative actions there (group admin layer, ban layer off), or, through community authority, a writer whose block carries a valid `auth` (below). Leave withdrawal does NOT apply to reading, neither to the `group-key` block nor to the grant that gave its writer authority (the distributor check reads the root's `group-admin` records and C's admin layer without leave withdrawal), so an admin who later leaves or is demoted, or a creator whose leave withdraws the grant, never takes back the epochs already handed out and history stays readable. For reading, a demotion cuts only the writers it lists, at their listed cut (a writer it omits keeps the read validity of its blocks, though it is cut at 0 for everything else), and a root demotion never reaches a block that sorts before it in causal order, whatever its cuts; grants still give nothing to unlisted writers and bans still apply by their cuts. So no demotion, honest or not, makes history unreadable. Sending keeps the strict layer: a first block counts for Eligibility, coverage and the key-holding and power records of "Removals" only when its writer passes the ordinary check with demotions cutting every writer, so no member ever sends under the epoch of a writer that lost power. That check also applies leave withdrawal to the grants: a grant that a creator's leave withdrew keeps the blocks it authorized readable but gives no sending power. A block valid for sending must also come from its diary's current generation: when a leave of that diary sorts before the block in causal order, its authority must come from a grant that saw every leave of that diary sorting before the block ("Entitled writers"; for the creator, a writer whose rejoin names the latest such leave). A block that sorts before every leave of its diary keeps its authority, so the epochs an admin opened before leaving stay valid history, and a block that covers a leave has that leave as an edge, so it needs a grant written after the leave. The writer that left, any other writer of that diary from before the leave (whatever grant or community power still lists it) and a creator's root after the creator left keep reading power only, so a block they write after a rejoin, wherever its `at` sorts it, never becomes an epoch anyone sends under, never counts as a recipient or authority record and is never reported by `onKey`. A rejoined distributor regains sending power on its new writer once a grant that saw its leave lists that writer. Other blocks are ignored. By owner decision, the creator of C and the admins of C (`C.admins`) distribute and receive keys in every private group of C through community authority, like group admins, while they hold that power; losing it is an admin change (below). For sending, an epoch must also pass "Eligibility". -**Community distributors (`auth`).** A `group-key` block K in a G writer w of diary d, when d is not G's creator and no G grant covers (w, seq), counts through the community only when `K.auth` is not null and names a position `{ writer: cw, seq: s }`, where `s` (an integer, at least 1) is the length of the distributor's own CW when it wrote the block (the next position in that CW, like a cut), such that, with C's fold, `cwOk` holds (`auth` must cite the writer's own CW): `C.cwDiary.get(cw) == d` (C's root for C's creator) and `s` is at most the held length of that core (the length it cites exists here). Reading (`dist`) needs `cwOk` and, for C's creator, the root bound, or, for anyone else, some counted `on: true` root grant of C that lists cw with a cut at or below s (demotions, C bans and leave withdrawal are ignored for reading, as history must never become unreadable: a grant and a later demotion can both cut at the same length when the admin wrote nothing in its CW in between). The community gate never withdraws a key block's read validity either. Sending (`send`) needs the strict layer: d's grant in C's current admin layer on at (cw, s) and C's ban layer off there, `C.admins.has(d)`, `live(w)`, and the cited CW holding C authority now (in C's `authorityWriters`: a CW retired, banned per CW or demoted never sends, even when its diary is still a C admin through another device). `auth` is ignored (as null) on a block whose writer holds G authority, so G authority wins; a block with `auth: null` from a diary without G authority is not counted. `auth` is a validity citation, not an ordering edge: an edge into another core's chain would let a CW position pin a block's place in G's order. Nodes write `auth = { writer: ownCW, seq: ownCW.length }` when they distribute through community authority, and `null` otherwise. Citing the length matters: a grant's cut is the CW's observed length L, so an admin promoted at L that wrote nothing in its CW since cites L, which the grant covers; a demotion's cut M turns the strict layer off from M on, so a demoted admin's blocks are never sent under. A demoted admin can still write blocks that read as valid by citing an old position; nobody sends under them, and only its own posts could use them, which a community ban or a hide covers. Everything else (kid, commit, tag, basis, rekey-pending, `onKey` only for full epochs) is unchanged. +**Community distributors (`auth`).** A `group-key` block K in a G writer w of diary d, when d is not G's creator and no G grant covers (w, seq), counts through the community only when `K.auth` is not null and names a position `{ writer: cw, seq: s }`, where `s` (an integer, at least 1) is the length of the distributor's own CW when it wrote the block (the next position in that CW, like a cut), such that, with C's fold, `cwOk` holds (`auth` must cite the writer's own CW): `C.cwDiary.get(cw) == d` (C's root for C's creator) and `s` is at most the held length of that core (the length it cites exists here; of a cut CW, the length its writer signed, "Sparse copies"). Reading (`dist`) needs `cwOk` and, for C's creator, the root bound, or, for anyone else, some counted `on: true` root grant of C that lists cw with a cut at or below s (demotions, C bans and leave withdrawal are ignored for reading, as history must never become unreadable: a grant and a later demotion can both cut at the same length when the admin wrote nothing in its CW in between). The community gate never withdraws a key block's read validity either. Sending (`send`) needs the strict layer: d's grant in C's current admin layer on at (cw, s) and C's ban layer off there, `C.admins.has(d)`, `live(w)`, and the cited CW holding C authority now (in C's `authorityWriters`: a CW retired, banned per CW or demoted never sends, even when its diary is still a C admin through another device). `auth` is ignored (as null) on a block whose writer holds G authority, so G authority wins; a block with `auth: null` from a diary without G authority is not counted. `auth` is a validity citation, not an ordering edge: an edge into another core's chain would let a CW position pin a block's place in G's order. Nodes write `auth = { writer: ownCW, seq: ownCW.length }` when they distribute through community authority, and `null` otherwise. Citing the length matters: a grant's cut is the CW's observed length L, so an admin promoted at L that wrote nothing in its CW since cites L, which the grant covers; a demotion's cut M turns the strict layer off from M on, so a demoted admin's blocks are never sent under. A demoted admin can still write blocks that read as valid by citing an old position; nobody sends under them, and only its own posts could use them, which a community ban or a hide covers. Everything else (kid, commit, tag, basis, rekey-pending, `onKey` only for full epochs) is unchanged. **Reading.** `buildGroup` takes an optional `keys`: `null` or `{ writers: Set, unseal(sealed), onKey? }`. `writers` are the reader's own writer keys in this group (current and retired writers, and G when the reader's device created the group: they all share the device's box key). `unseal(sealed)` takes one `sealed` base64 string and returns its raw 48-byte plaintext `epochKey || tag` (`crypto_box_seal_open` with the device key pair) or null; it never checks the tag. The fold requires exactly 48 bytes, checks the tag against the block's G and kid ("`group-key`"), copies the 32-byte key and zero-fills both the 48 bytes and the buffer `unseal` returned. The fold calls `unseal` only on entries naming one of `writers` inside counted `group-key` blocks; an epoch whose first block is valid only for reading (or withdrawn) is opened only when some counted, not withdrawn envelope names its kid, since nothing else could be read under it; an opened key counts only when `keyCommit(key, G, kid)` equals the epoch's first-block `commit` (compared in constant time); an entry that fails is ignored and later entries and blocks are still tried, so for each kid the fold keeps the committed key from the first block in causal order that opens to it. It never puts a key in its result. The optional `onKey(kid, key, recipients)` is called once per epoch the fold accepted and opened whose first block is valid for sending and not withdrawn by a leave (after the tag and commitment checks, in causal order of the first block; an epoch valid only for reading, such as a departed or demoted distributor's, stays readable inside the fold but is never reported, so it is never handed on), with a `{ writer, seq }` copy of the kid, a COPY of the 32-byte key that the caller owns and must zero-fill, and `recipients`, the sorted writer keys listed by every block of that epoch valid for sending and not withdrawn by a leave; an error it throws is ignored and changes nothing. A node gets the keys it may hand on (history, rotation, sealing) only from this hook, never by recomputing distributors itself. An opened envelope becomes its inner record at the envelope's `{ writer, seq }`, with the inner record's ordering edges plus the envelope's `kid` edge (so it never sorts before its epoch's first block), and with the authority, leave withdrawal and membership generation (the leaves of its diary before it) the envelope had in the plaintext pass below; an envelope that stays sealed is a node with only its writer edge and its `kid` edge, and is never shown. An opened `group-react` is the exception: its envelope stays that sealed node and the reaction is folded beside the order ("Reactions"), without counting in `sealed`. The fold runs twice. The plaintext pass keeps every envelope sealed and alone decides membership and moderation (`members`, `requests`, `admins`, `bans`, `leaves`, `community`, `writers`) and the epoch machinery (Distributors, Removals, Entitled writers, Eligibility); the opened pass, under that same authority, only places content (posts, hides, reactions, the name) and orders it, so decrypted `seen`, `replyTo` and `target` edges never change who is a member or who may act. The fold is pure and order independent for a given `keys`; two readers with different keys may see different posts and order, never different membership. The name of a private group is the `name` of the last opened, valid, not withdrawn `group-meta` in the root by root seq; a sealed meta whose `join` is not `approval` is skipped, and `join` is always `approval`. Without a key the name is null. @@ -2136,7 +2138,7 @@ Mirrors "Seeder media admission" (section 5) with the same invariants, plus the - **Root G.** Admitted by the same steps as a writer (including quarantine, persistence before the open and the held blocklist) when its block 0 is a valid `group-header` and some served, gated, unblocked diary has a live join for G (the creator's join with `writer == group` is the normal path). **Root binding.** The header's diary counts as the root's creator (and as support) only when that diary is served, not dropped, not blocked and has its own `on: true` `group-join` for writer = G. Otherwise the root is supported only by real member joins, and `/index/group/` answers `creator: null` (also during the startup window before the creator's scan). **Support:** a served root stays while at least one served, unblocked diary keeps it: the creator if it bound the root, a diary with a bound writer in it, or a live join. Withdrawing every join does not drop a root that still has a bound writer. At startup a root does not survive just because its creator's diary is kept: decisions that find a root unsupported are deferred until the startup scans of the served diaries end, and it is judged after them; an unsupported root is then dropped with reason `unbound`. **Blocked or revoked creator:** a served root is purged for a blocked diary, or one whose certificate issuer is revoked, only when that diary BOUND the root (`bound: true`: its header diary has its own `on: true` join for writer = G). Reason `blocked` (also during the startup window), the root stays refused at admission and at startup while its creator stays gone, even while members support the group. A root whose header merely names a blocked diary that never bound it has no creator and stays while members support it. A creator that is both blocked and from a revoked issuer has that issuer recorded too (`revokedCreators`, and `goneRoots` for its bound roots), and a stored issuer is never replaced by null: lifting only the block keeps its bound roots refused, also after a restart, while the issuer stays revoked. A creator dropped or refused for a revoked issuer is remembered with that issuer (`revokedCreators`, a moderation outcome, also across restarts) and judged against the CURRENT revoked-issuer set: adopting a list that lifts the revocation lifts it. An announce of a revoked-issuer certificate is not remembered at all, not even in memory: only a diary's own block-0 verdict (refused at startup, or dropped as revoked-issuer while served) feeds `revokedCreators`, a `goneRoots` issuer and the refusal of a root at admission for a revoked creator, so a root whose creator was never served on this seeder is not refused for a revoked issuer (the root only names that diary). A repeated revoked announce of the same key from one peer counts as a bad announce; the first does not. A purged root is remembered with its creator and, for a revocation, its issuer (`goneRoots`) and refused again while that creator stays gone; lifting the block or revocation lets the same join admit it, and a root admitted again after that keeps `bound`. A diary admitted with a verifying certificate clears its stale revoked entries (`revokedCreators` and the `goneRoots` entries naming it with an issuer). A root whose header names a currently blocked diary, or a diary whose known issuer is revoked, is refused at admission even without a purge record (a root admitted before the block stays until its binding says otherwise). Roots bound before this version get `bound` only after their creator's next scan. A root its creator bound stays listed on `/index/group/` (and in the writer pages) under that creator diary while members keep it served after the creator is dropped for a reason other than moderation (quota, fork, seeder full), also after a restart; `header.diary` stays `null` then, so no creator authority is shown. Members' own writers stay under the rules above. **Issuer kept with the binding.** A bound root's binding keeps the issuer of its creator's block-0 certificate (`issuer`, taken at the creator's scan). When that issuer is revoked under the current list and the creator is not served, the bound root is purged with reason `revoked-issuer` and remembered like any revoked-issuer drop (`revokedCreators`, `goneRoots`), also at startup and also when the creator left first for another reason, such as its quota, so a member that keeps the root served cannot outlast the revocation. **Header fields kept.** While any core of group G is served or being admitted, the seeder keeps the verified header fields of its root (`kind`, `visibility`, `join`, `community`, `member`, and `name` for a public root) in `groupHeaders`, also after the root was purged or refused or its block 0 is withheld, and drops them once no core of G is left or G is blocked; past the bound the record is not kept. A persisted record without `community` and `member` is a protocol v1 leftover and is dropped at start. - **Protocol v2 admission and scan.** A header whose validator returns `legacy` is `header-invalid` (no new wire reason), and the startup re-gate purges every stored v1 root and writer with that reason. A group root must carry a hex64 `community`; the seeder never checks a writer's `member` or any authorization (the fold does). CWs are admitted like any writer and take part in the writer pages. For a served community root the index folds `group-meta`, `group-admin` and `group-create` from the root only (creates are indexed from C's root alone; a CW is not scanned for them, `group-create` being `root-only` there). A public group root is scanned for `group-meta` only (one scan at a time across group roots, skipping blocks without the bytes `group-meta`) and a private root is not scanned. **Denial bounds.** `revokedCreators` and `goneRoots` never evict an entry still in force: when a map reaches 4096 entries the lifted ones go first, and `admitted.json` may hold up to 8192 entries in each. While 4096 or more denials are in force in either map, or after a denial found no room (past 8192), the seeder refuses every group root with the group-rejected reason `denials-full` and logs `{ event: 'warning', reason: 'denials-full' }`, so a purged root cannot return for lack of memory. -- **Cut writers** (#350). For now a writer its diary cut (frozen, released, or live with a cut once the frozen slots or the releases are spent) is held whole, like any writer: its full log under the whole-log quota (at admission, after the open and on every append) and the caps above (32 cores per diary and group, 1024 per diary). The fold counts only block 0, the leaves and the key blocks past a cut, but every node reads a writer from block 0 and stops at its first missing block, so a seeder that cleared the other blocks past the cut would hand a fresh reader a copy that folds without the leave. Keeping only what the fold counts waits for client nodes that read such sparse copies. The cut is not persisted with the binding: the diary's scan recomputes cuts and releases. +- **Cut writers** (#350, owner decision). A writer its diary cut (frozen, released, or live with a cut once the frozen slots or the releases are spent: `groupWalkCut` over the slot walk) keeps its blocks below the cut, block 0 always, and from the cut on only its `group-leave` and `group-key` blocks, the ones the fold still counts there and readers read past a hole ("Diary cuts on own writers", "Sparse copies"). A released writer (cut 0) is a stub: block 0, its leaves and its key blocks. Its download stops at the cut; the blocks past it, up to `MAX_LOG_BLOCKS`, are fetched `CUT_CHUNK` = 256 at a time, read for their type only (`decodeBlock`, never validated) and cleared unless kept. Hypercore keeps the tree nodes the proofs of the kept blocks need, so a reader verifies each against the signed length. The binding in admitted.json carries `cut`, `clearedFrom` and `clearedTo` (the range past the cut already read) and is persisted before a chunk is cleared; the next serve reads that range again and clears what it still holds beyond the kept blocks, so a crash in between leaves nothing extra for good. The quota counts what is kept: at admission and at startup, the blocks below the cut (`min(max(cut, 1), length)` against `maxLogBlocks`); while the whole log is over the quota, the blocks below the cut and those kept past it, and their bytes, against `maxLogBlocks` and `maxLogBytes`, counted as they are read. A writer past it is dropped with `quota` and its join is refused as at admission. The caps above count a stub like any writer. The diary's scan gives the cut: a lower one applies at once (also to a core still opening), a higher one or none only once the scan caught up with the diary (a cut restored from admitted.json whose record was deleted, for example), and then the download reaches the new cut, or the whole log, so what clearing removed is fetched again. A cut restored at startup clears nothing new until the scan confirms it. A join refused before (`quota`, `header-invalid`) is tried again once per lower cut. A writer of a private group whose header has no box (the one a node replaced, "Writers named by a node") is `header-invalid` while counted and kept as a stub once released, so its block 0 stays available to readers, which reject it by its header ("Limits"); if its release goes away it is judged again and dropped. - **Drops** (purge before un-listing, runtime order of media (e), each logs `{ event: 'group-rejected', key, diary, group, reason }`): the diary is dropped or blocked (D's writers go with it, with `blocked` or the diary's own reason; the root stays while supported), the root key is in the blocklist snapshot (`group-blocked`, for the root and every writer of that group; the blocklist apply loop also walks the group map, because it otherwise walks only `_served`), a fork (`fork`), the quota (`quota`), an unsupported root (`unbound`), a root whose creator (the header diary, when it bound the root) is blocked or has a revoked issuer (`blocked`) or a failed header re-check (`header-invalid`). NEVER on a `group-leave`, never on a join turned off and never on a cut: a cut or released writer stays held whole ("Cut writers"). - **Retention.** Index withdrawal and membership end are distinct, as defined above. Admitted writer and root cores stay served and listed across either event (the index only marks the writer `left`), preserving the moderation records of departed members. - **Startup** re-gates listed group cores like listed media cores (a core whose binding is gone, or whose root is blocked, is purged first), applying the same per-diary counts to persisted bindings (32 per group); for bindings written before the cap that count follows iteration order, not earliest join. A cut writer, a released one included, is restored whole (quota on the whole log), and the scans recompute cuts and releases from the diaries. `seed list` prints the group cores indented under their diary (`grupo (grupo ) blocos bytes`), `seed drop` purges them as auxiliaries, and `stats` adds `groupCores`. diff --git a/docs/verification/feat-writer-slot-sparse.md b/docs/verification/feat-writer-slot-sparse.md new file mode 100644 index 00000000..9b827cf6 --- /dev/null +++ b/docs/verification/feat-writer-slot-sparse.md @@ -0,0 +1,110 @@ +# Escritores cortados guardados e lidos esparsos (branch feat/writer-slot-sparse) + +Parte da #350, item 8a, passo 8a-3, sobre `feat/writer-slot-walk`. Decisão do dono (não reaberta aqui): o semeador guarda, dentro da cota normal, as saídas e os blocos de chave de um escritor cortado e um toco de cada escritor liberado. + +## O que mudou + +Dobra (`packages/core/lib/group.js`): + +- `bindCores`: um escritor conta até o primeiro seq que nenhuma cópia tem e, depois desse buraco, só as saídas (`group-leave`) e os blocos de chave (`group-key`), e só quando o buraco está no corte ou depois dele (`max(corte, 1)`). Uma cópia esparsa de um escritor cortado dobra como a inteira; uma cópia parcial, cortada ou não, dobra como o prefixo de sempre. +- Cada cópia traz `signed` (o comprimento que o escritor assinou). Um bloco de chave cujo `auth` cita um escritor de comunidade (CW) cortado vale quando `auth.seq` não passa desse comprimento (`heldSeq = max(lastSeq, held, signed - 1)`), então um CW aparado no corte lê o histórico que a cópia inteira lê. + +Leitores: + +- Nó core: `readHeldBlocks` em `packages/core/lib/store.js` (o prefixo contíguo e, depois do primeiro buraco, só as saídas e os blocos de chave guardados, até `MAX_LOG_BLOCKS`); `readCoreParts` em `member.js` lê os escritores assim e passa `signed: core.length`. As raízes continuam lidas como prefixo. +- Nó do navegador: `readHeldBlocks` em `packages/web-member/lib/node-store.js`; em `member-ops.js`, `writerCoreState` para os escritores, o carimbo da memória da dobra (`groupStamp = replicaStamp`) conta os blocos guardados depois do comprimento contíguo, `signed` em `foldGroup` e `foldCommunity`, e `foldCommunity` passa a receber as marcas de corte dos diários. +- Serviço de contas: `apps/cli/lib/account/pages-groups.js` lê os escritores listados com `readHeldBlocks` e passa `signed`. + +Semeador (`packages/core/lib/seeder.js`): + +- Escritor cortado (`groupWalkCut` sobre a caminhada): o download para no corte; os blocos depois dele, até `MAX_LOG_BLOCKS`, são buscados de 256 em 256 (`CUT_CHUNK`), lidos só pelo tipo e limpos, menos saídas e blocos de chave (`_cutGroupCore`). O Hypercore guarda os nós da árvore que as provas dos blocos mantidos pedem. +- Liberado (corte 0): toco com o bloco 0, as saídas e os blocos de chave. +- A ligação em `admitted.json` guarda `cut`, `clearedFrom` e `clearedTo`, gravada antes de limpar cada pedaço; o próximo serviço relê essa faixa e limpa o que sobrou de uma queda no meio. +- Cota: na admissão e na partida, só os blocos abaixo do corte; com o núcleo inteiro acima da cota, os blocos abaixo do corte mais os mantidos depois dele, e os bytes deles. Passou: sai com `quota` e a entrada fica recusada, como na admissão (senão cada varredura buscaria de novo). +- Corte menor aplica na hora (também com o núcleo ainda abrindo); maior ou nenhum, só depois que a varredura alcançou o diário, e então o que a limpeza tirou é buscado de novo. Um corte restaurado na partida não limpa nada novo antes de a varredura confirmá-lo. +- Uma entrada recusada antes (`quota`, `header-invalid`) é tentada de novo uma vez a cada corte menor. +- Escritor sem caixa num grupo privado: `header-invalid` enquanto contado, toco depois de liberado (`groupVerdict(..., released)`, na admissão, na partida, no portão e na reverificação de caixa). Se a liberação some, é julgado de novo e sai. + +Documentos: `docs/protocol.md` (bit `CAP_WRITER_CUTS`, entrada de `buildGroup` e `buildCommunity` com `signed`, "Slot walk" e "Writers named by a node" sobre o sem caixa, regra do `auth.seq` em "Diary cuts on own writers" e em "Community distributors", parágrafo novo "Sparse copies", "Cut writers" do semeador reescrito). `docs/rpc.md` não muda: nenhum método nem resultado mudou. + +## Como verificar + +Cada arquivo sozinho, a partir da raiz, com `node `, e `npx standard` nos arquivos mudados: + +| Arquivo | Testes | Asserções | +|---|---|---| +| `packages/core/test/seeder-groups-cuts.test.js` (testes 7 a 14 novos) | 14/14 | 327/327 | +| `packages/core/test/sync-groups-sparse.test.js` (novo, DHT local) | 1/1 | 12/12 | +| `packages/core/test/group-private.test.js` | 128/128 | 898/898 | +| `packages/core/test/group-member-6.test.js` | 10/10 | 72/72 | +| `packages/web-member/test/group-memo.test.js` | 11/11 | 60/60 | +| `packages/web-member/test/group-ops.test.js` | 57/57 | 436/436 | +| `packages/core/test/seeder-private.test.js` | 4/4 | 34/34 | +| `packages/core/test/seeder-groups.test.js` | 43/43 | 262/262 | +| `packages/core/test/seeder-groups-3.test.js` | 6/6 | 33/33 | +| `packages/core/test/seeder-groups-waits.test.js` | 12/12 | 61/61 | +| `packages/core/test/seeder-communities.test.js` | 9/9 | 59/59 | +| `apps/cli/test/pages-groups.test.js` | 3/3 | 153/153 | +| `apps/cli/test/account-pages-groups.test.js` | 1/1 | 49/49 | + +O teste de ponta a ponta: A cria a comunidade e dois grupos; B escreve nos dois, congela o escritor de G1 no comprimento (dois posts e a saída depois do corte) e libera o de G2 (um post e a saída). Depois que o semeador guarda os dois esparsos, R, novo, sincroniza só com ele: guarda W1 abaixo do corte mais a saída e W2 como toco, e a comunidade e os dois grupos (info, membros e leitura) dobram igual a quando R recebe as cópias inteiras de B. + +`apps/cli/lib/account/pages-groups.js` não tem teste próprio da leitura esparsa: usa o mesmo `readHeldBlocks` e a mesma dobra testados acima, e hoje o `fetch` dele marca o grupo como `incomplete` com uma cópia esparsa (ponto (b) abaixo), então a diferença ainda não aparece para quem usa. + +Não rodados aqui, por regra do brief: a suíte inteira e a demo (a sessão que abriu esta roda a CI de entrega). + +## Prova de regressão + +Cada correção foi tirada e o arquivo rodado de novo. O brittle para no primeiro erro lançado, então nos casos de "tempo esgotado" o resto do arquivo não roda. + +Dobra (`group.js`, `packages/core/test/group-private.test.js`), com o `bindCores` de `feat/writer-slot-walk`: 126/128 testes, 894/898 asserções. Falham "the signed length keeps the history readable" e "the sparse CW folds like the whole one" (CW aparado no corte e bloco de chave que o cita), "a hole below the cut: the prefix" e "an uncut writer: the prefix". + +Nó core (`packages/core/test/group-member-6.test.js`), com `readBlocks` no lugar de `readHeldBlocks` em `readCoreParts`: 9/10, 71/72, falha "the sparse copy folds like the whole one". + +Nó do navegador (`packages/web-member/test/group-memo.test.js`): + +- `groupCoreState` em vez de `writerCoreState` para os escritores: 9/11, 56/60, falham "the leave past the hole counts" e "the sparse copy folded like the whole one" (grupo) e "the leave past the cut counts" e "the sparse copy folded like the whole one" (comunidade). +- `foldCommunity` sem as marcas de corte: 10/11, 58/60, falham "the leave past the cut counts" e "the sparse copy folded like the whole one" (comunidade). +- Carimbo antigo (`contiguous`, `length`, `fork`): a conferência da memória lança "stored fold differs from a fresh one". + +Semeador (`packages/core/test/seeder-groups-cuts.test.js`), uma mutação por vez: + +| Mutação | Resultado | +|---|---| +| M1 `_cutGroupCore` retorna na hora (nada é limpo) | tempo esgotado "W cortado em 3" (teste 7) | +| M2 cota da admissão sobre o núcleo inteiro | tempo esgotado "group-admitted" (teste 12) | +| M3 bytes mantidos depois do corte fora da cota | tempo esgotado "group-rejected ... quota" (teste 12) | +| M4 sem `caught && cut !== now` (corte que some não é aplicado) | tempo esgotado "W buscado de novo" (teste 11) | +| M5 sem `&& !released` no veredito do sem caixa | tempo esgotado "group-admitted" (teste 14) | +| M6 recusa não levantada por um corte menor | tempo esgotado "group-admitted" (teste 14) | +| M7 sem `groupRefused.set` ao sair pela cota do que guarda | 13/14, 326/327, falha "a writer dropped for what it keeps is not fetched again" | +| M8 `_groupCutScan` depois do teste de servido | o teste 13 estoura os 150000 ms | +| M9 veredito da partida sem a liberação | 13/14, 325/327, falham "N served at ready" e "not purged" | +| M10 `cut`, `clearedFrom` e `clearedTo` fora da ligação | falha "from its persisted cut" (teste 10), depois tempo esgotado "sem corte persistido" (teste 11) | + +Ponta a ponta (`packages/core/test/sync-groups-sparse.test.js`): + +- E1, `readBlocks` em `readCoreParts` (nó core lê os escritores como prefixo): 0/1, 11/12, falha "the sparse copies fold like the whole ones". O semeador e as cópias de R estão certos; só a dobra difere. +- E2, M1 no semeador (escritores cortados guardados inteiros): tempo esgotado "run condition", o semeador nunca fica esparso. + +## Decisões tomadas aqui + +- A regra do buraco mora na dobra (`bindCores`), não só nos leitores: assim o nó core, o nó do navegador e o serviço de contas dobram igual mesmo com cópias diferentes do mesmo escritor, e um bloco depois de um buraco abaixo do corte nunca conta. +- Os escritores são lidos com `readHeldBlocks`; as raízes continuam com o prefixo (nunca são cortadas por um diário). +- No nó do navegador, `foldCommunity` passa a receber as marcas de corte: sem elas, um CW cortado guardado esparso dobrava diferente da cópia inteira. +- O semeador grava `cut`, `clearedFrom` e `clearedTo` antes de limpar; uma recontagem a cada serviço limpa o que uma queda deixou. A faixa lida fica numa peça só: um corte menor lê para baixo a partir de `clearedFrom` antes de seguir para cima. +- Um corte restaurado do `admitted.json` serve na hora, mas não limpa nada novo antes de a varredura do diário confirmá-lo (`cutConfirmed`). Corte menor aplica na hora (a varredura só acrescenta marcas, então nunca fica acima do corte do diário); corte maior ou ausente só depois de a varredura alcançar o diário (`scanCaughtUp`). +- Uma entrada recusada é tentada de novo uma vez a cada corte menor que a caminhada dá ao escritor: um corte faz caber na cota quem não cabia, e a liberação faz o sem caixa valer como toco. O custo é no máximo uma busca do bloco 0 por registro de corte. +- Um escritor que sai por causa do que guarda fica com a entrada recusada (`groupRefused`), como uma recusa na admissão; sem isso ele seria admitido, buscado e derrubado a cada varredura. +- A cota do que fica guardado usa `_overQuota` (núcleo inteiro) como atalho: só conta blocos e bytes quando o núcleo inteiro já passa da cota. Os bytes abaixo do corte são lidos um a um só nesse caso. +- O fragmento de changelog de `feat/writer-slot-walk` (que diz que o semeador guarda o escritor inteiro) não foi mexido: é de outro PR; o deste PR diz o comportamento novo. + +## Riscos residuais e pontos em aberto + +- (a) Meta de `sync --once` (nó core e nó do navegador): `groupUnmet` quer `max(anunciado, core.length)` blocos contíguos de cada escritor alheio, e o dono de um escritor cortado espera o semeador listar o comprimento inteiro (`pushed`). Com o semeador esparso, nenhum dos dois chega lá, e um `--once` num grupo com escritor cortado só termina pelo tempo limite. Padrão recomendado: dar o escritor por completo quando o contíguo alcança o anunciado e todo bloco que um par conectado anuncia (bitfield) abaixo do comprimento assinado está guardado; para o próprio escritor, comparar com o corte do diário. Não feito aqui: está fora do escopo do brief e mexe nas duas sincronizações. +- (b) Serviço de contas: `fetch` em `pages-groups.js` dá uma cópia por completa só com o contíguo no comprimento, então um grupo com escritor cortado esparso fica `incomplete` e entrar, postar ou selar pela Página responde `indisponivel`. Padrão recomendado: completa quando todo bloco que o semeador anuncia está guardado. +- (c) As dobras de catálogo (comunidade no índice) não recebem marcas; leem o prefixo, o que continua seguro. Sem mudança. +- (d) As cotas do lado do leitor usam o comprimento assinado (já era assim). +- (e) Banda do semeador: os blocos depois do corte, até `MAX_LOG_BLOCKS`, são baixados só para ler o tipo. Os pedaços de 256 limitam o que fica guardado além do mantido, não o que trafega. +- (f) Item 4 do lado do leitor: as sincronizações (core e navegador) ainda recusam com `header-invalid` o cabeçalho sem caixa num grupo privado antes de guardá-lo, então um leitor que nunca o teve continua com a dobra `incomplete`, embora o semeador agora mantenha o bloco 0. Padrão recomendado: guardar só o bloco 0 de um escritor sem caixa de grupo privado quando a caminhada do diário dele o libera, e deixar a dobra rejeitá-lo pelo cabeçalho. É uma mudança grande nas duas sincronizações; não feita aqui. +- O teste de ponta a ponta não confere o resultado `ok` do `sync --once` do leitor, por causa de (a). From 7ab4f7bd8f097cf8da3c1bf141b0fa2efd7c456c Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 01:53:22 -0300 Subject: [PATCH 08/25] feat(sync): end a once run with a sparse cut writer A foreign writer held up to a hole past what the seeder lists counts as caught up once every block a synced peer advertises below its signed length is held (core/lib/sparse-held.js). An own writer the diary cut is pushed once the seeder holds it up to the cut. --- packages/core/lib/group.js | 14 ++++ packages/core/lib/sparse-held.js | 67 +++++++++++++++++++ packages/core/lib/sync.js | 53 +++++++++++++-- packages/core/test/sync-groups-sparse.test.js | 9 ++- 4 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 packages/core/lib/sparse-held.js diff --git a/packages/core/lib/group.js b/packages/core/lib/group.js index 8c29f897..5217c905 100644 --- a/packages/core/lib/group.js +++ b/packages/core/lib/group.js @@ -869,6 +869,19 @@ function walkHasRoom (joins, marks, G, extra = []) { } } +// The cut a diary's slot walk for G gives its writer W, as a seeder keeps W +// (seeder.js groupWalkCut): 0 once released, its lowest cut while counted, +// null with none. `entries`: the diary's records as read ({ seq, record }). +// Never throws. +function diaryWriterCut (entries, G, W) { + try { + const w = slotWalk(groupJoinHistory(entries, G), groupCutHistory(entries, G), G) + if (w.released.has(W)) return 0 + if ((w.live.has(W) || w.frozen.has(W)) && w.cut.has(W)) return w.cut.get(W) + } catch {} + return null +} + function slotWalkResult (s, gap = false, bound = false) { const live = new Set() const frozen = new Set() @@ -3929,6 +3942,7 @@ module.exports = { groupCutHistory, groupConfirmHistory, slotWalk, + diaryWriterCut, slotWalkState, replacedWriterCut, slotWalkStep, diff --git a/packages/core/lib/sparse-held.js b/packages/core/lib/sparse-held.js new file mode 100644 index 00000000..66fb2c4f --- /dev/null +++ b/packages/core/lib/sparse-held.js @@ -0,0 +1,67 @@ +'use strict' + +// A seeder keeps a cut writer sparse (#350, docs/protocol.md "Sparse +// copies"): its blocks below the cut, then only its leaves and key blocks. +// A reader holding such a copy is as caught up as its peers let it be once +// every block a connected peer advertises below the signed length is held. +// Hypercore (10 and 11) keeps what a peer advertises per replicator Peer +// (core.peers, as deletion.js advertisedBy reads it): its contiguous range, +// then its bitfield. Past that range the bitfield only arrives as the answer +// to the want a download range sends, so until a page of it is here a block +// the peer holds there reads as not advertised: such a peer has not +// answered yet, and a goal that trusted it would end before the block came. +// Browser-safe: no Node built-ins. + +// RemoteBitfield's page size, the same in Hypercore 10 and 11. +const BITFIELD_PAGE = 32768 + +// Whether `peer` has told this core what it holds of [its contiguous +// length, end): it has synced, and either holds nothing past its contiguous +// range below end or a page of its bitfield covering that range arrived. +function peerAnswered (peer, end) { + if (peer.remoteSynced !== true) return false + const from = peer.remoteContiguousLength + const to = Math.min(end, peer.remoteLength || 0) + if (from >= to) return true + for (let p = Math.floor(from / BITFIELD_PAGE); p * BITFIELD_PAGE < to; p++) { + if (peer.remoteBitfield.getBitfield(p * BITFIELD_PAGE) === null) return false + } + return true +} + +// The first block of [from, end) `peer` advertises, or -1. +function nextAdvertised (peer, from, end) { + if (from < peer.remoteContiguousLength) return from < end ? from : -1 + const bits = peer.remoteBitfield + if (typeof bits.firstSet === 'function') { + const i = bits.firstSet(from) + return i >= 0 && i < end ? i : -1 + } + for (let i = from; i < end; i++) if (bits.get(i) === true) return i + return -1 +} + +// -> true once some connected peer of `core` has synced, every synced peer +// has answered for the blocks below `end` (peerAnswered) and every block +// below `end` one of them advertises is held. A peer that has not synced +// yet is not waited for. Never throws (false on any error). +async function advertisedHeld (core, end) { + try { + const peers = [...core.peers].filter((p) => p.remoteSynced === true) + if (peers.length === 0) return false + for (const peer of peers) if (!peerAnswered(peer, end)) return false + const from = core.contiguousLength + for (const peer of peers) { + let i = nextAdvertised(peer, from, end) + while (i !== -1) { + if (!(await core.has(i))) return false + i = nextAdvertised(peer, i + 1, end) + } + } + return true + } catch { + return false + } +} + +module.exports = { advertisedHeld, peerAnswered, BITFIELD_PAGE } diff --git a/packages/core/lib/sync.js b/packages/core/lib/sync.js index 90eb0fe4..210ea860 100644 --- a/packages/core/lib/sync.js +++ b/packages/core/lib/sync.js @@ -73,7 +73,8 @@ const { pageLifecycle, originOf, verifyClose, verifyMoved } = require('./page-li const { certForKey, seederCertForKey } = require('./cert') const { createBlocklistState, verifySnapshot, compareSnapshots } = require('./blocklist') const { attachBlocklist } = require('./blocklist-net') -const { validateGroupHeader, validateWriterHeader, buildCommunity, groupJoinHistory, groupConfirmHistory, MAX_GROUP_WRITERS, MAX_GROUPS_PER_DIARY, MAX_GROUP_WRITERS_PER_DIARY, MAX_GROUPS_PER_COMMUNITY } = require('./group') +const { validateGroupHeader, validateWriterHeader, buildCommunity, groupJoinHistory, groupConfirmHistory, diaryWriterCut, MAX_GROUP_WRITERS, MAX_GROUPS_PER_DIARY, MAX_GROUP_WRITERS_PER_DIARY, MAX_GROUPS_PER_COMMUNITY } = require('./group') +const { advertisedHeld } = require('./sparse-held') const { attachGroups, GROUPS_LIMIT_MAX, GROUPS_REQUEST_TIMEOUT_MS, MESSAGE_BURST: GROUPS_BURST, MESSAGE_RATE: GROUPS_RATE } = require('./groups-net') const { presentState, @@ -136,6 +137,9 @@ const MAX_WRITER_PAGES = Math.ceil(MAX_GROUP_WRITERS / GROUPS_LIMIT_MAX) // the socket dropped for 'message-rate' (that would cut diary replication). const GROUP_REQUEST_BURST = Math.floor(GROUPS_BURST / 8) const GROUP_REQUEST_RATE = GROUPS_RATE / 2 +// A foreign writer held past its prefix is checked again this often while +// its goal waits on what its peers advertise (sparse-held.js). +const SPARSE_CHECK_MS = 100 // Page items whose diary or join seq a gate disproved, per seeder, before // this run stops paging that seeder. const MAX_BAD_HINTS = 32 @@ -2130,7 +2134,7 @@ async function runSync ({ continue } seen.add(g.group) - ownWriters.set(g.writer, { group: g.group, core }) + ownWriters.set(g.writer, { group: g.group, core, cut: null }) groupList.push({ group: g.group, writer: g.writer, nonce: g.nonce }) groupSet.add(g.group) if (g.kind === 'community' || (g.writer === g.group && await ownRootKind(core) === 'community')) ownCommunities.add(g.group) @@ -2151,6 +2155,16 @@ async function runSync ({ } catch {} } } + // A writer the own diary cut is pushed whole at the cut: the seeders + // keep it sparse past it (docs/protocol.md "Sparse copies"). + if (ownWriters.size > 0) { + try { + const entries = await readAll(diary) + for (const [w, own] of ownWriters) if (w !== own.group) own.cut = diaryWriterCut(entries, own.group, w) + } catch { + log('aviso: falha ao ler os cortes do diário') + } + } // A joined group's root stored by an earlier run names its community // before any seeder lists it again. for (const g of groupList) { @@ -2618,7 +2632,10 @@ async function runSync ({ if (it && it.length > pushed) pushed = it.length } const own = ownWriters.get(g.writer) - if (own.core.length > pushed) unmet.push(g.writer + ':' + own.core.length + '/' + pushed) + // A writer its diary cut is kept to the cut, block 0 always: past it + // a seeder holds only what it keeps, never the whole length. + const goal = own.cut === null ? own.core.length : Math.min(own.core.length, Math.max(own.cut, 1)) + if (goal > pushed) unmet.push(g.writer + ':' + goal + '/' + pushed) } for (const [key, advertised] of groupTargets) { if (ownWriters.has(key)) continue @@ -2628,9 +2645,12 @@ async function runSync ({ const active = e && e.state === 'active' const ours = active ? e.core.contiguousLength : 0 const want = active ? Math.max(advertised, e.core.length) : advertised + // A copy past the advertised prefix with a hole (a writer a seeder keeps + // sparse) is met once its peers' advertised blocks are all held. + const short = !active || (ours < want && (ours < advertised || !sparseMet(e))) // Until the main store's block 0 is compared with the gated one, ending // the run would skip that comparison and the growth it reports. - if (!active || ours < want || e.expected !== null) unmet.push(key + ':' + ours + '/' + want) + if (short || e.expected !== null) unmet.push(key + ':' + ours + '/' + want) if (active) { const d = groupDiaries.get(e.diary) if (d && !d.failed && (d.present < d.want || d.expected !== null)) unmet.push(e.diary + ':' + d.present + '/' + d.want) @@ -2639,6 +2659,29 @@ async function runSync ({ return unmet } + // Whether foreign writer e, held up to a hole past what its seeders list, + // holds every block a connected peer advertises below its signed length, + // from the last check at its present length and prefix (sparse-held.js). + // A missing or stale check runs again shortly; evaluate() follows it. + function sparseMet (e) { + const s = e.sparse + if (s && s.length === e.core.length && s.contiguous === e.core.contiguousLength && s.done) return true + if (e.sparseTimer || ended) return false + e.sparseTimer = later(() => { + e.sparseTimer = null + const core = e.core + if (ended || core === null || e.state !== 'active') return + const length = core.length + const contiguous = core.contiguousLength + track(advertisedHeld(core, Math.min(length, maxLogBlocks)), (done) => { + if (e.core !== core) return + e.sparse = { length, contiguous, done } + evaluate() + }, noop) + }, SPARSE_CHECK_MS) + return false + } + function groupSockets (peer) { const out = [] if (!peer.socket.destroyed) out.push(peer.socket) @@ -3009,6 +3052,8 @@ async function runSync ({ diaryRaw: null, // D's block 0, judged again when an issuer is revoked wasStored: false, purging: null, + sparse: null, // { length, contiguous, done }: the last advertisedHeld check + sparseTimer: null, accepted: false // gated as a writer of this group (now or in an earlier run) } groupCores.set(item.key, e) diff --git a/packages/core/test/sync-groups-sparse.test.js b/packages/core/test/sync-groups-sparse.test.js index 1666a3c3..79b5ab7f 100644 --- a/packages/core/test/sync-groups-sparse.test.js +++ b/packages/core/test/sync-groups-sparse.test.js @@ -117,9 +117,14 @@ test('a fresh member fed only by a seeder folds a sparse frozen writer and a rel await runUntil(b, bootstrap, sparse) t.alike(await blocksHeld(seeder._groupServed.get(w1).core, 0, leave1.seq + 1), [...Array(cut1).keys(), leave1.seq], 'the seeder keeps W1 below its cut and its leave') t.alike(await blocksHeld(seeder._groupServed.get(w2).core, 0, leave2.seq + 1), [0, leave2.seq], 'and W2 as a stub') + // The owner's --once counts each cut writer pushed at its cut. + const rb = await syncOnce(bootstrap, b, { timeoutMs: 30000 }) + t.ok(rb.result.ok, 'B\'s run ends ok before its deadline: ' + rb.result.errors.join('; ')) - // R, fresh, syncs with the seeder alone (B is offline). - await syncOnce(bootstrap, r, { timeoutMs: 30000 }) + // R, fresh, syncs with the seeder alone (B is offline); its --once ends + // once it holds every block the seeder advertises. + const rr = await syncOnce(bootstrap, r, { timeoutMs: 30000 }) + t.ok(rr.result.ok, 'R\'s run ends ok before its deadline: ' + rr.result.errors.join('; ')) const copies = await heldCopies(r, [w1, w2]) t.alike(copies[w1], { length: leave1.seq + 1, contiguous: cut1, held: [...Array(cut1).keys(), leave1.seq] }, 'R holds W1 sparse') t.alike(copies[w2], { length: leave2.seq + 1, contiguous: 1, held: [0, leave2.seq] }, 'and W2 as a stub') From 1a7148b000224779e191fc6c909bdda6db43c5a7 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 01:53:33 -0300 Subject: [PATCH 09/25] feat(web-member): end a once run with a sparse cut writer The browser node's goals follow the core node: a foreign writer held past the seeder's prefix is met once every advertised block is held, and an own writer the diary cut counts pushed at the cut. --- packages/web-member/lib/sync.js | 59 ++++++++- .../test/sync-groups-sparse.test.js | 114 ++++++++++++++++++ 2 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 packages/web-member/test/sync-groups-sparse.test.js diff --git a/packages/web-member/lib/sync.js b/packages/web-member/lib/sync.js index 47387b59..f172d5cd 100644 --- a/packages/web-member/lib/sync.js +++ b/packages/web-member/lib/sync.js @@ -339,7 +339,7 @@ const { attachAdmission, MAX_INVENTORY } = require('@varanda/core/lib/admission' const { createBlocklistState, compareSnapshots } = require('@varanda/core/lib/blocklist') const { attachBlocklist } = require('@varanda/core/lib/blocklist-net') const { attachGroups, GROUPS_LIMIT_MAX } = require('@varanda/core/lib/groups-net') -const { validateGroupHeader, validateWriterHeader, validateGroupRecord, groupJoinHistory, groupConfirmHistory, groupJoins, buildCommunity, diaryGroupKeys, MAX_GROUP_WRITERS, MAX_GROUP_WRITERS_PER_DIARY, MAX_GROUPS_PER_COMMUNITY } = require('@varanda/core/lib/group') +const { validateGroupHeader, validateWriterHeader, validateGroupRecord, groupJoinHistory, groupConfirmHistory, groupJoins, buildCommunity, diaryGroupKeys, diaryWriterCut, MAX_GROUP_WRITERS, MAX_GROUP_WRITERS_PER_DIARY, MAX_GROUPS_PER_COMMUNITY } = require('@varanda/core/lib/group') const { presentState, presentLength, @@ -348,6 +348,7 @@ const { downloadExceptPages, pageTargets } = require('@varanda/core/lib/deletion') +const { advertisedHeld } = require('@varanda/core/lib/sparse-held') const { memoryStorage } = require('./storage') const { guardedWsStream } = require('./ws-stream') const store = require('./node-store') @@ -405,6 +406,9 @@ const GROUP_PAGE_MS = 30000 // While an own group core waits for its push confirmation, the pages are // asked again at most this often (the channel allows 2 messages a second). const GROUP_PUSH_PAGE_MS = 1000 +// A foreign writer held past its prefix is checked again this often while +// its goal waits on what its peers advertise (core sparse-held.js). +const SPARSE_CHECK_MS = 100 // Writer page requests on one link are spaced this much, all groups and // every re-page together (the seeder refills 2 messages a second). const GROUP_REQUEST_MS = 1000 @@ -1559,7 +1563,10 @@ async function syncRun ({ const heldAtStart = new Set() // group cores stored when the run started const notCommunities = new Set() // parent keys whose root is a group header: never paged again this run let changedTimer = null - const fold = { cursor: 1, seed: null, ops: new Map(), seen: false, running: false, again: false, joinEntries: [], joinCursor: 1, scanPending: false, holes: new Set(), readFails: new Map(), retryTimer: null, holeFilled: false, scanLogged: false } + const fold = { cursor: 1, seed: null, ops: new Map(), seen: false, running: false, again: false, joinEntries: [], cutEntries: [], joinCursor: 1, scanPending: false, holes: new Set(), readFails: new Map(), retryTimer: null, holeFilled: false, scanLogged: false } + // The own diary's group-join and group-cut records as of its last scan + // (adoptDiaryGroups), for the cut of each own writer; null before it. + let ownCutEntries = null const diaryWriters = new Set() // writers the own diary binds on:true, seen by adoptDiaryGroups const diaryChase = new Map() // new writer of an adopted root -> its re-page timer let diaryLive = new Map() // writer -> root the own diary binds now, as of the last scan @@ -3486,13 +3493,23 @@ async function syncRun ({ }) } + // How much of own group core m a seeder must hold: its length, or for a + // writer the own diary cut, up to the cut (block 0 always), since past it + // a seeder keeps only the leaves and key blocks (docs/protocol.md "Sparse + // copies"). + function pushTarget (m, length) { + if (m.key === m.group || ownCutEntries === null) return length + const cut = diaryWriterCut(ownCutEntries, m.group, m.key) + return cut === null ? length : Math.min(length, Math.max(cut, 1)) + } + function checkMedia () { const l = seederLink() if (l === null || ended) return for (const m of pushCores()) { const length = m.core.length if (length < 1 || m.pushed === length) continue - if (m.group ? seederHoldsGroup(m, l, length) : seederHoldsMedia(m, l, length)) { + if (m.group ? seederHoldsGroup(m, l, pushTarget(m, length)) : seederHoldsMedia(m, l, length)) { m.pushed = length emit({ event: m.event, key: m.key, length }) } @@ -3569,6 +3586,8 @@ async function syncRun ({ stopFork: null, stopQuota: null, retryTimer: null, + sparse: null, // { length, contiguous, done }: the last advertisedHeld check + sparseTimer: null, reason: null, // why it failed hints: new Set() // 'group:diary:seq' listings whose gate failed on the hint } @@ -5173,7 +5192,10 @@ async function syncRun ({ continue } const want = Math.max(ge.core.length, advertised) - if (ge.core.contiguousLength < want) unmet.push(ge.key + ':' + ge.core.contiguousLength + '/' + want) + const ours = ge.core.contiguousLength + // A copy past the advertised prefix with a hole (a writer a seeder keeps + // sparse) is met once its peers' advertised blocks are all held. + if (ours < want && (ours < advertised || !sparseMet(ge))) unmet.push(ge.key + ':' + ours + '/' + want) } // A community's catalogue scan in flight may still add roots to page. for (const [c, st] of catalogueScans) if (st.running || st.again) unmet.push(c + ':catalogue') @@ -5198,6 +5220,30 @@ async function syncRun ({ return unmet } + // Whether foreign writer ge, held up to a hole past what the seeder lists, + // holds every block a connected peer advertises below its signed length, + // from the last check at its present length and prefix (core + // sparse-held.js). A missing or stale check runs again shortly; + // evaluate() follows it. + function sparseMet (ge) { + const s = ge.sparse + if (s && s.length === ge.core.length && s.contiguous === ge.core.contiguousLength && s.done) return true + if (ge.sparseTimer || ended) return false + ge.sparseTimer = later(() => { + ge.sparseTimer = null + const core = ge.core + if (ended || core === null || ge.state !== 'active') return + const length = core.length + const contiguous = core.contiguousLength + track(advertisedHeld(core, Math.min(length, maxLogBlocks)), (done) => { + if (ge.core !== core) return + ge.sparse = { length, contiguous, done } + evaluate() + }, noop) + }, SPARSE_CHECK_MS) + return false + } + // ---- the follow fold (see the header) ---- // Replays the own diary's follow records from the cursor up to its @@ -5241,6 +5287,7 @@ async function syncRun ({ fold.holes.delete(seq) fold.readFails.delete(seq) if (record !== null && record.type === 'group-join') fold.joinEntries.push({ seq, record }) + if (record !== null && record.type === 'group-cut') fold.cutEntries.push({ seq, record }) } catch { // Once per scan: a diary of failing blocks must not flood the log. if (!ended && !fold.scanLogged) { @@ -5373,6 +5420,7 @@ async function syncRun ({ let added = false const adopted = new Set() const entries = fold.joinEntries.filter((e) => !deleted.has(e.seq)) + ownCutEntries = [...entries, ...fold.cutEntries.filter((e) => !deleted.has(e.seq))] const keys = diaryGroupKeys(entries) for (const key of keys) { if (adoptDiaryRoot(key)) { @@ -5394,6 +5442,8 @@ async function syncRun ({ } const l = seederLink() if (added && l !== null && l.groups !== null) pageGroups(l) + // An own writer the diary cut may be pushed far enough now. + checkMedia() } catch { if (!ended) log('aviso: falha ao ler os grupos do diário') } @@ -5437,6 +5487,7 @@ async function syncRun ({ } const record = validRecordCopy(entry.record, entry.seq) if (record !== null && record.type === 'group-join') fold.joinEntries.push({ seq: entry.seq, record }) + if (record !== null && record.type === 'group-cut') fold.cutEntries.push({ seq: entry.seq, record }) } fold.joinCursor = Math.max(1, state.present) } diff --git a/packages/web-member/test/sync-groups-sparse.test.js b/packages/web-member/test/sync-groups-sparse.test.js new file mode 100644 index 00000000..813f6173 --- /dev/null +++ b/packages/web-member/test/sync-groups-sparse.test.js @@ -0,0 +1,114 @@ +// A seeder keeps a writer its diary cut sparse (#350, docs/protocol.md +// "Sparse copies"): below the cut, then only its leaves and key blocks. The +// owner's --once counts that writer pushed at its cut, and a fresh browser +// member fed by that seeder alone ends its --once holding the sparse copy. +const test = require('brittle') +const b4a = require('b4a') +const { testnet, until } = require('./helpers/net') +const { generateAssociationKeyPair } = require('@varanda/core/lib/cert') +const { groupCutRecord } = require('@varanda/core/lib/records') +const grp = require('@varanda/core/lib/group') +const store = require('../lib/node-store') +const ops = require('../lib/member-ops') +const { member, seederOverWs, indexed, sync, once, held } = require('./helpers/sync-groups') + +async function blocksHeld (core, from, to) { + const out = [] + for (let i = from; i < to; i++) if (await core.has(i)) out.push(i) + return out +} + +// -> { length, contiguous, held } of a stored group core. Only with no run +// of m live (helpers/sync-groups.js held()). +async function heldCopy (m, key) { + const core = await store.openGroupCoreReadOnly(m.storage, key) + try { + return { length: core.length, contiguous: core.contiguousLength, held: await blocksHeld(core, 0, core.length) } + } finally { + await core.close() + } +} + +async function ownWriter (m, group) { + const e = (await store.readGroupWriters(m.storage)).find((x) => x.group === group) + return e === undefined ? null : e.key +} + +// A leave on m's own writer in group, written past the member ops as an +// offline device would: group.leave also unbinds the writer in the diary, +// and a seeder lists a writer its diary unbound with that record, which a +// reader's gate refuses (it proves an on: true join). +async function offlineLeave (m, group) { + const entry = (await store.readGroupWriters(m.storage)).find((x) => x.group === group) + const core = await store.openGroupWriter(m.storage, await m.ctx.groupKeyPair(new Uint8Array(b4a.from(entry.nonce, 'hex')))) + try { + await store.appendGroupRecord(core, grp.groupLeave({ seen: null, cuts: [{ writer: entry.key, seq: core.length }] }), {}) + return core.length + } finally { + await core.close() + } +} + +test('a sparse cut writer: the owner\'s and a fresh reader\'s --once end ok (#350)', async (t) => { + t.timeout(240000) + const { bootstrap } = await testnet(t) + const assocKp = generateAssociationKeyPair() + const { seeder, url } = await seederOverWs(t, assocKp, bootstrap) + const a = await member(assocKp, 'Ana') + const b = await member(assocKp, 'Bia') + const r = await member(assocKp, 'Rui') + const { community } = await ops.communityCreate(a.ctx, { name: 'Bairro', join: 'open' }) + const { group } = await ops.groupCreate(a.ctx, { name: 'Horta', join: 'open', community }) + await ops.groupPost(a.ctx, { group, text: 'oi' }) + t.ok((await once(a, url, [])).ok, 'creator pushed') + await until(() => indexed(seeder, group, group), { name: 'root indexed' }) + + await store.addGroupReplica(b.storage, group) + t.ok((await once(b, url, [])).ok, 'B fetched the group and its community') + t.is((await ops.communityJoin(b.ctx, { community })).state, 'member', 'B joined the community') + t.is((await ops.groupJoin(b.ctx, { group })).state, 'member', 'and the group') + await ops.groupPost(b.ctx, { group, text: 'antes' }) + const w = await ownWriter(b, group) + const cut = (await held(b, w)).length + await ops.groupPost(b.ctx, { group, text: 'depois' }) + const length = await offlineLeave(b, group) + const diary = await store.openDiary(b.storage, b.ctx.keyPair) + try { + await store.appendRecord(diary, groupCutRecord({ group, writer: w, cut })) + } finally { + await diary.close() + } + + // B's run pushes its diary and writers; it stops once the seeder holds W + // sparse. + const ac = new AbortController() + const run = sync(b, url, [], { signal: ac.signal }) + try { + await until(async () => { + const e = seeder._groupServed.get(w) + const s = seeder._groups.get(w) + if (!e || !s || e.cutting || s.cut !== cut || s.clearedTo !== length) return false + return (await blocksHeld(e.core, 0, length)).length === cut + 1 + }, { name: 'the seeder holds W sparse', timeoutMs: 60000 }) + } finally { + ac.abort() + await run + } + t.alike(await blocksHeld(seeder._groupServed.get(w).core, 0, length), [...Array(cut).keys(), length - 1], 'the seeder keeps W below its cut and its leave') + + // The owner's --once counts W pushed at its cut. + const rb = await once(b, url, []) + t.ok(rb.ok, 'B\'s run ends ok before its deadline: ' + JSON.stringify(rb.errors)) + + // R, fresh, syncs with the seeder alone; its --once ends once it holds + // every block the seeder advertises. + await store.addGroupReplica(r.storage, group) + const rr = await once(r, url, []) + t.ok(rr.ok, 'R\'s run ends ok before its deadline: ' + JSON.stringify(rr.errors)) + t.alike(await heldCopy(r, w), { length, contiguous: cut, held: [...Array(cut).keys(), length - 1] }, 'R holds W sparse') + const members = (await ops.groupMembers(r.ctx, { group })).members.map((m) => m.member) + t.absent(members.includes(b.ownHex), 'the leave past the hole counts') + const texts = (await ops.groupRead(r.ctx, { group })).items.map((p) => p.text) + t.ok(texts.includes('antes'), 'the post below the cut counts') + t.absent(texts.includes('depois'), 'the post past the cut does not') +}) From 895371f233f9fc0ffb5e4223ae20a6f66a06b5b9 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 01:56:06 -0300 Subject: [PATCH 10/25] feat(account): count a sparse cut writer whole in a page's group fold A listed writer's fetch ends once every block the seeder advertises below its signed length is held, so a group with a writer the seeder keeps sparse no longer answers indisponivel to a page's join or post. --- apps/cli/lib/account/pages-groups.js | 32 ++++- apps/cli/test/pages-groups-sparse.test.js | 138 ++++++++++++++++++++++ 2 files changed, 165 insertions(+), 5 deletions(-) create mode 100644 apps/cli/test/pages-groups-sparse.test.js diff --git a/apps/cli/lib/account/pages-groups.js b/apps/cli/lib/account/pages-groups.js index 03033c97..e22f44fb 100644 --- a/apps/cli/lib/account/pages-groups.js +++ b/apps/cli/lib/account/pages-groups.js @@ -51,6 +51,7 @@ const { attachAdmission } = require('@varanda/core/lib/admission') const { attachGroups, GROUPS_LIMIT_MAX } = require('@varanda/core/lib/groups-net') const { generateNoiseKeyPair } = require('@varanda/core/lib/net') const { groupJoinRecord, latestProfile, validateRecord, MAX_LOG_BLOCKS } = require('@varanda/core/lib/records') +const { advertisedHeld } = require('@varanda/core/lib/sparse-held') const { latestUsername } = require('@varanda/core/lib/username') const groups = require('@varanda/core/lib/group') const { keyTag, keyCommit, wipe } = require('@varanda/core/lib/group-crypto') @@ -67,6 +68,9 @@ const NOISE_FILE = 'noise-key.json' const MAX_REPLICAS = 1024 const MAX_WRITER_PAGES = Math.ceil(groups.MAX_GROUP_WRITERS / GROUPS_LIMIT_MAX) const FETCH_TIMEOUT_MS = 10000 +// A listed writer held past the seeder's prefix is checked this often for +// the blocks the seeder advertises (core sparse-held.js). +const SPARSE_POLL_MS = 50 const BACKOFF_MIN_MS = 1000 const BACKOFF_MAX_MS = 60000 const HANDSHAKE_TIMEOUT_MS = 10000 @@ -604,16 +608,34 @@ function createPagesGroups ({ dir, writer, pagesStore, keys, association, blockl // The core at the length the seeder offers, held (release it), or null // when nothing arrived in time. `complete` is false while a block up to - // min(length, MAX_LOG_BLOCKS) is missing. - async function fetch (key, held) { + // min(length, MAX_LOG_BLOCKS) is missing, or for a `sparse` core (a listed + // writer, which a seeder keeps sparse once its diary cut it, #350) while + // a block the seeder advertises below that is missing. + async function fetch (key, held, sparse = false) { const core = await hold(key) held.push(key) await timeout(core.update({ wait: true }).catch(noop), fetchTimeoutMs) if (core.length === 0) return null const end = Math.min(core.length, MAX_LOG_BLOCKS) - await timeout(core.download({ start: 0, end }).done().catch(noop), fetchTimeoutMs) + const range = core.download({ start: 0, end }).done().catch(noop) + if (!sparse) { + await timeout(range, fetchTimeoutMs) + if (core.contiguousLength === 0) return null + return { core, complete: core.contiguousLength >= end } + } + // The range never completes past a seeder's hole: it ends once every + // block the seeder advertises is here. + const deadline = Date.now() + fetchTimeoutMs + let done = false + range.then(() => { done = true }) + let complete = false + for (;;) { + complete = core.contiguousLength >= end || (core.contiguousLength > 0 && await advertisedHeld(core, end)) + if (complete || done || closed || Date.now() >= deadline) break + await timeout(range, Math.min(SPARSE_POLL_MS, Math.max(0, deadline - Date.now()))) + } if (core.contiguousLength === 0) return null - return { core, complete: core.contiguousLength >= end } + return { core, complete } } function localWriter (key) { @@ -699,7 +721,7 @@ function createPagesGroups ({ dir, writer, pagesStore, keys, association, blockl // replica of the same key would never get a channel, so read it here. let got = localWriter(it.key) try { - if (got === null) got = await fetch(it.key, held) + if (got === null) got = await fetch(it.key, held, true) } catch { incomplete = true continue diff --git a/apps/cli/test/pages-groups-sparse.test.js b/apps/cli/test/pages-groups-sparse.test.js new file mode 100644 index 00000000..3c7e92ee --- /dev/null +++ b/apps/cli/test/pages-groups-sparse.test.js @@ -0,0 +1,138 @@ +'use strict' + +// apps/cli/lib/account/pages-groups.js with a writer a seeder keeps sparse +// (#350, docs/protocol.md "Sparse copies"): its blocks below the diary cut, +// then only its leaves and key blocks. The page's fold counts such a copy +// whole once every block the seeder advertises is held. Its own file: the +// pages-groups.test.js runs already take most of the runner's per-file time. + +const test = require('brittle') +const b4a = require('b4a') +const crypto = require('hypercore-crypto') +const { testnet, tmpDir, until } = require('./helpers/net') +const { createIssuerCert, createIssuedCert, createUsernameRecord, generateAssociationKeyPair } = require('@varanda/core/lib/cert') +const { signRoster, issuerCountersign } = require('@varanda/core/lib/page') +const member = require('@varanda/core/lib/member') +const coreStore = require('@varanda/core/lib/store') +const { groupPost, groupLeave } = require('@varanda/core/lib/group') +const { groupCutRecord } = require('@varanda/core/lib/records') +const { person, exchange, startSeeder, syncOnce, ownWriter, runUntil } = require('../../../packages/core/test/helpers/sync-groups') +const { offlineAppend, writerEntries } = require('../../../packages/core/test/helpers/group-member') +const { startSeedWs } = require('../lib/seed-ws') +const { openPageKeys } = require('../lib/account/page-keys') +const { createPagesWriter } = require('../lib/account/pages-writer') +const { openPagesStore } = require('../lib/account/pages-store') +const { createPagesGroups } = require('../lib/account/pages-groups') + +const PAGE_KEYS = 'cd'.repeat(32) + +function hex (buf) { + return b4a.toString(buf, 'hex') +} + +let nonceCount = 0 +function nonce () { + nonceCount++ + return (String(nonceCount) + 'g'.repeat(43)).slice(0, 43) +} + +function newPage (ctx, keys, username) { + const { key, row } = keys.pending(nonce()) + keys.put(key, row) + const cert = createIssuedCert({ member: key }, ctx.issuerKp, ctx.issuerCert) + const usernameRecord = createUsernameRecord({ member: key, username }, ctx.issuerKp, ctx.issuerCert) + const owner = crypto.keyPair() + const fields = { page: key, gen: 1, owner: hex(owner.publicKey), admins: [], prev: null, at: new Date().toISOString(), nonce: nonce(), association: ctx.association } + const signature = signRoster(fields, owner) + const rosterRecord = issuerCountersign({ type: 'page-roster', ...fields, signature }, ctx.issuerKp, ctx.issuerCert) + return { key, owner: hex(owner.publicKey), args: { cert, usernameRecord, rosterRecord } } +} + +// Retries while the fold is still partial ('indisponivel'). +function settled (fn, name, timeoutMs) { + return until(async () => { + try { + return await fn() + } catch (err) { + if (err.code === 'indisponivel') return null + throw err + } + }, { name, timeoutMs, intervalMs: 500 }) +} + +async function blocksHeld (core, from, to) { + const out = [] + for (let i = from; i < to; i++) if (await core.has(i)) out.push(i) + return out +} + +test('pages-groups: a group with a writer the seeder keeps sparse is no longer indisponivel (#350)', async (t) => { + t.timeout(300000) + const { bootstrap } = await testnet(t) + const kp = generateAssociationKeyPair() + const association = hex(kp.publicKey) + const issuerKp = crypto.keyPair() + const ctx = { association, issuerKp, issuerCert: createIssuerCert({ issuer: hex(issuerKp.publicKey) }, kp) } + const seeder = await startSeeder(t, bootstrap, kp) + const server = await startSeedWs({ port: 0, seeder }) + t.teardown(() => server.close(), { force: true }) + const seederWs = 'http://127.0.0.1:' + server.port + + const a = await person(t, kp, 'Ana') + const b = await person(t, kp, 'Bia') + const { community: C } = await member.communityCreate(a, { name: 'Bairro', join: 'open' }) + const { group: G } = await member.groupCreate(a, { name: 'Praça', join: 'open', community: C }) + await member.groupPost(a, { group: G, text: 'oi' }) + await exchange(a, b) + await member.communityJoin(b, { community: C }) + await member.groupJoin(b, { group: G }) + await member.groupPost(b, { group: G, text: 'antes' }) + // W, B's writer in G, is frozen at its length: a post and a leave past + // the cut. + const w = ownWriter(b, G) + const cut = (await writerEntries(b, G)).length + await offlineAppend(b, G, groupPost({ text: 'depois' })) + const leave = await offlineAppend(b, G, groupLeave({ seen: null, cuts: [{ writer: w, seq: cut + 1 }] })) + const s = await coreStore.openStore(b.dataDir) + try { + await coreStore.appendRecord(await coreStore.openDiary(s), groupCutRecord({ group: G, writer: w, cut })) + } finally { + await s.close() + } + t.ok((await syncOnce(bootstrap, a)).result.ok, 'A pushes its diary and roots') + await runUntil(b, bootstrap, async () => { + const e = seeder._groupServed.get(w) + const st = seeder._groups.get(w) + if (!e || !st || e.cutting || st.cut !== cut || st.clearedTo !== leave.seq + 1) return false + return (await blocksHeld(e.core, 0, leave.seq + 1)).length === cut + 1 + }) + t.alike(await blocksHeld(seeder._groupServed.get(w).core, 0, leave.seq + 1), [...Array(cut).keys(), leave.seq], 'the seeder keeps W below its cut and its leave') + + const dir = tmpDir(t, 'varanda-account-') + const keys = openPageKeys({ dir, env: { VARANDA_PAGE_KEYS: PAGE_KEYS } }) + const pagesStore = openPagesStore(dir) + const writer = createPagesWriter({ dir, keys, association, seederWs }) + t.teardown(() => writer.close(), { force: true }) + await writer.ready() + const pg = createPagesGroups({ dir, writer, pagesStore, keys, association, seederWs }) + t.teardown(() => pg.close(), { force: true }) + await pg.ready() + await until(() => writer.connected() && pg.connected(), { name: 'links up', timeoutMs: 20000 }) + const page = newPage(ctx, keys, 'padaria') + await writer.createPage(page.args) + await pagesStore.create({ page: page.key, owner: page.owner }) + + t.is((await settled(() => pg.communityJoin(page.key, C), 'community join', 60000)).state, 'member', 'the page joins the community') + // Each try fetches W again; before #350's goal it waited out the fetch + // timeout and answered indisponivel every time. + t.is((await settled(() => pg.groupJoin(page.key, G), 'group join', 60000)).state, 'member', 'the page joins the group with W sparse') + // The seeder indexes the page's new writer first: its listing is partial + // meanwhile. + const posted = await settled(() => pg.groupPost(page.key, G, { text: 'pão quente' }), 'group post', 60000) + t.ok(Number.isSafeInteger(posted.seq), 'and posts') + const read = await pg.groupRead(page.key, G) + t.is(read.incomplete, false, 'its fold is whole') + const texts = read.items.map((p) => p.text) + t.ok(texts.includes('antes'), 'the post below the cut counts') + t.absent(texts.includes('depois'), 'the post past the cut does not') +}) From d487e5e71c18f156c228be401878ef923f73f42f Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 02:06:55 -0300 Subject: [PATCH 11/25] docs: describe completion goals with sparse cut writers Completion goals in protocol.md now cover a writer held sparse, an own writer measured against its diary cut, and the account service's fetch. The verification file adds this round's tests, mutation proofs and open points; the changelog adds what a user sees. --- docs/changelog/feat-writer-slot-sparse.md | 2 +- docs/protocol.md | 2 +- docs/verification/feat-writer-slot-sparse.md | 61 +++++++++++++++++--- 3 files changed, 54 insertions(+), 11 deletions(-) diff --git a/docs/changelog/feat-writer-slot-sparse.md b/docs/changelog/feat-writer-slot-sparse.md index 19f3826b..4a76b34d 100644 --- a/docs/changelog/feat-writer-slot-sparse.md +++ b/docs/changelog/feat-writer-slot-sparse.md @@ -1,3 +1,3 @@ ### Alterado -- Os semeadores passam a guardar só o que conta de um escritor cortado pelo diário (PR `feat/writer-slot-sparse`, parte da #350): os blocos abaixo do corte e, a partir dele, só as saídas do grupo e os blocos de chave. De um escritor liberado (corte 0) fica um toco: o bloco 0, as saídas e os blocos de chave. A cota conta o que fica guardado. O nó core, o nó do navegador e o serviço de contas leem um escritor além do primeiro buraco (só as saídas e os blocos de chave), então quem só fala com semeadores vê o grupo como quem tem as cópias inteiras. Um bloco de chave que cita um escritor de comunidade cortado vale pelo comprimento que o escritor assinou. O escritor sem caixa trocado num grupo privado, depois de liberado, também fica como toco no semeador. Conferência em `docs/verification/feat-writer-slot-sparse.md`. +- Os semeadores passam a guardar só o que conta de um escritor cortado pelo diário (PR `feat/writer-slot-sparse`, parte da #350): os blocos abaixo do corte e, a partir dele, só as saídas do grupo e os blocos de chave. De um escritor liberado (corte 0) fica um toco: o bloco 0, as saídas e os blocos de chave. A cota conta o que fica guardado. O nó core, o nó do navegador e o serviço de contas leem um escritor além do primeiro buraco (só as saídas e os blocos de chave), então quem só fala com semeadores vê o grupo como quem tem as cópias inteiras. Um bloco de chave que cita um escritor de comunidade cortado vale pelo comprimento que o escritor assinou. O escritor sem caixa trocado num grupo privado, depois de liberado, também fica como toco no semeador. O `sync --once` (nó core e navegador) termina com um escritor guardado esparso quando todo bloco que os pares anunciam está guardado, e o dono dá o escritor cortado por enviado no corte; entrar, postar ou selar pela Página num grupo com escritor esparso não responde mais `indisponivel`. Conferência em `docs/verification/feat-writer-slot-sparse.md`. diff --git a/docs/protocol.md b/docs/protocol.md index 45ba31c8..6da6f7ae 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -2180,7 +2180,7 @@ Added to section 7 "Seeder index route", under the same caps, Host check, 405 an - **Round 13, core node.** (1) A new writer key of a group is refused (`group-skipped` reason `quota`) while the group's foreign writer cores in the admission table plus those stored on disk, including writers kept after a quota stop, reach `MAX_GROUP_WRITERS` (`runSync` option `maxGroupWriters`; the group root is no writer slot and is never refused by this cap); a stored writer keeps its place, and a refused key is gated again once a place frees. (2) Block-0 divergences of group writers become one hard-error line `bloco 0 divergente em escritores de grupo: ()` (the result is still not ok), and a run keeps at most 32 hard-error strings. - **Membership and purges.** A fetched historical join block proves binding only. Current membership comes from the writer fold above. A stored writer core is purged only for an ACCEPTED writer (one that passed the gate): when D is blocked or its issuer is revoked, on a fork, or when its local block 0 differs from the gated one. A bad hint from a seeder never purges an unaccepted core. A blocked group root fails and purges every foreign core of that group with reason `blocked`, in the core node and in the browser alike; the member's own cores are never touched, and what is stored can still be read locally until it is purged. - **Binding diary release (core node).** A group writer's binding diary is downloaded and kept in the main store only while something needs it: a follow holds it, a writer in the admission table names it, or a stored foreign writer or root names it (the census is rebuilt from disk at start). When a writer is purged, retires or is evicted and nothing else needs its diary, the node stops downloading the diary and purges it from the main store. A writer stopped for quota keeps its diary (and its blocks). At run start a binding diary recorded in `group-certs.json` that no stored core names and that no follow holds is purged. At core node start, a member of any group also purges an unfollowed, unlisted diary unless a stored group writer or root names it or `group-certs.json` records it; that cleanup always rebuilds the census first, also when `groups.json` and `group-certs.json` are empty, and never purges a group core this node can write. A `group-join` block that its author deleted (section 2 "Clearing deleted bytes") withdraws the writer it bound: the core node drops the writer from the run (`group-skipped` reason `withdrawn`) without purging its blocks, and the group goal stops waiting for it. At browser node start, a stored replica that a stored group core binds (or that the run holds as a binding diary) stays indexed and is never purged by the unfollowed-replica cleanup. -- **Completion goals** (`--once` and the browser's once runs). Each group goal runs once per run and is met when (a) every writer page of every joined group was fetched (not `incomplete`) or the channel timed out, (b) every admitted foreign writer, and the diary that binds it up to its join, reached the length its seeder advertised, exactly like followed replicas (on the core node a core admitted through quarantine counts only after the block 0 its main store received was compared with the gated one), and (c) own writers were pushed to the length the seeder advertises for them (browser: confirmed by `group-pushed { key, length }`); all bounded by the run's deadline. Against a seeder without `varanda/groups/1` (groups unsupported) the browser's own group pushes are not once-run goals, so a once run finishes. The goal of a binding diary is met at its present length (section 2 "Clearing deleted bytes"), so a deleted post no longer keeps a once run open, and join blocks after a cleared hole are read. Group writer and root cores keep their own tracking, since no delete records exist for them. Goals G1 to G3 of section 6 are unchanged. The browser's write gate for a group write confirms the own writer's length, not only the diary. For each community the goals are C's root, every page of its CWs, and the binding diaries, as for groups. +- **Completion goals** (`--once` and the browser's once runs). Each group goal runs once per run and is met when (a) every writer page of every joined group was fetched (not `incomplete`) or the channel timed out, (b) every admitted foreign writer, and the diary that binds it up to its join, reached the length its seeder advertised, exactly like followed replicas (on the core node a core admitted through quarantine counts only after the block 0 its main store received was compared with the gated one), and (c) own writers were pushed to the length the seeder advertises for them (browser: confirmed by `group-pushed { key, length }`); all bounded by the run's deadline. A writer a seeder keeps sparse ("Sparse copies") never reaches its signed length contiguously, so (b) also holds for a foreign writer held up to the advertised length and past it with a hole once every connected peer that synced has answered for the blocks below the signed length (up to `MAX_LOG_BLOCKS`) and every block one of them advertises there is held (`advertisedHeld` in `sparse-held.js`, checked again every `SPARSE_CHECK_MS` = 100 ms while the goal waits). Past a peer's contiguous length its bitfield arrives only as the answer to a download range's want, so a peer counts as having answered once a page of its bitfield covering that range arrived. (c) for an own writer the own diary cut asks the seeder to hold it up to `max(cut, 1)` (`diaryWriterCut` over the diary's slot walk: 0 once released), never past the length the device holds; the core node reads the cuts when the run opens its writers, the browser at each scan of its diary. The account service's group fold for a page (`apps/cli/lib/account/pages-groups.js`, section 12) counts a listed writer whole on the same rule, so a group with such a writer is no longer partial (`indisponivel`): its fetch ends, before `FETCH_TIMEOUT_MS`, once every block the seeder advertises below the signed length is held (polled every 50 ms). Roots and diaries keep the contiguous rule. Against a seeder without `varanda/groups/1` (groups unsupported) the browser's own group pushes are not once-run goals, so a once run finishes. The goal of a binding diary is met at its present length (section 2 "Clearing deleted bytes"), so a deleted post no longer keeps a once run open, and join blocks after a cleared hole are read. Group writer and root cores keep their own tracking, since no delete records exist for them. Goals G1 to G3 of section 6 are unchanged. The browser's write gate for a group write confirms the own writer's length, not only the diary. For each community the goals are C's root, every page of its CWs, and the binding diaries, as for groups. - **Parent paging.** For every local group root held the node reads its header's `community`, and for every local community its key. For each community C it pages C on the groups channel (the root and its CWs), gates each CW like a group writer (a valid header with `group` = C and `member: null`, then the binding diary's block 0 and join block), stores the CWs and their binding diaries, and pages only the roots of C's catalogue (the `/index/community/` rows are a hint, the fold verifies) so that `community.info` can name them. Each root is judged by its own block 0 under a role fixed for the run: a key joined as a group must be a `group` root, a key reached as a parent must be a `community` root, and a `v: 1` header is `header-invalid`; the `group-skipped` reason `not-community` (a parent that is not a community root; the node's own verdict, the seeder is not charged) is kept. A run holds at most groups x 512 plus communities x 512 cores. A node folding a group passes `buildGroup` the parent's input (C's root, its CWs and their joins). The core sync emits `community-parent { key, group }` (`key` the community root C, `group` the group G) the first time a held group root names a community this node does not already hold, and `group-parent { key: C, group: G }` once per local group whose held root names C, held or not (the session uses it to fan `groups.changed` out from C's growth to its groups; the CLI prints neither). - **Diary-bound roots (browser).** Besides kv `groupWriters` and kv `groupReplicas`, the browser sync pages every root its own diary binds now as a replica: the newest `group-join` per writer still `on: true`, among the writers the slot walk counts per group ("Limits"), the first 64 groups by live join seq (`diaryGroupKeys`, the seeder's live-join rule). It reads them at run start from the stored diary and again whenever the own diary is caught up with the seeder, so a second device of the same passkey (same diary, fresh storage) rebuilds its communities and groups from the diary. A writer the diary newly binds to a root already adopted is paged at once (that group alone, ahead of the pass in progress or in a pass of its own), and again after waits of 2, 5 and 10 s (each counted from the previous page, so at about 2, 7 and 17 s) while the seeder does not list it yet; the writers of the run's first scan are not chased, the regular pass asks for them. A diary block the read could not get (not downloaded yet, or a read that failed) is kept, read again on every later scan, when it downloads and, after a failed read, on a short timer, and never holds back the joins after it. A once run does not end while that read of the own length is pending: the roots it adopts are paged before the run's goals are met. Such a root's writers are gated like any replica's, a catalogue child the diary joined is not paged root only (one already paged root only in the run is paged again in full), and a community among them is paged in full with its catalogue. Its own header decides its kind: another root's header naming it as its community never makes it a parent key, so it is never refused as `not-community`. A root no join binds now (every device left) is not adopted, a blocked key never is, and a key with no root at the seeder is dropped by the probe rule and not asked for again in that run. Nothing is appended, no writer is created and nothing is listed in kv for an adopted key until its block 0 is stored (the gate then lists it in kv `groupReplicas`). The core node is unchanged: its diary is per device (section 1), so another device never shares it. - **Automatic device enrolment (browser).** After a sync, a device of the same passkey enrols by itself in each diary-bound root it holds and has no kv `groupWriters` entry for, while the fold says the diary is a `member` there and the diary binds fewer than `MAX_GROUP_WRITERS_PER_DIARY` - 2 (6) writers in it: the last two slots, which the seeder never releases, are spent only by an explicit join. It writes what `community.join` or `group.join` writes (a new writer naming the diary's latest leave, and the diary's public `group-join`), communities first, then a group only when this device's community writer is admitted in C (an open community or its creator; in an approval community the diary's admission covers the new community writer, "Admits list their writers", and its groups follow on a later run, once the fold shows it admitted). Geral is planned like any group of C with its own checks: a diary that left Geral is not put back, and Geral's last two slots stay reserved too. A kv writer the diary never bound (an interrupted join) is bound again, reusing that writer only when this device derives its key from the entry's nonce; otherwise the root is skipped (`foreign`), and an explicit join writes a new writer instead. Only a writer this device derived is ever reused by a join. A root that is left, banned, requested, blocked, unheld, an orphan, not joined by the own diary (an opened link) or not foldable yet (no pending join) is skipped, and no admit or grant is written automatically (the fold's coverage of a new device by the diary's admission writes nothing). Rules in [rpc.md](rpc.md) 5.63. The core node is unchanged (its diary is per device, section 1). diff --git a/docs/verification/feat-writer-slot-sparse.md b/docs/verification/feat-writer-slot-sparse.md index 9b827cf6..720d2e93 100644 --- a/docs/verification/feat-writer-slot-sparse.md +++ b/docs/verification/feat-writer-slot-sparse.md @@ -25,7 +25,15 @@ Semeador (`packages/core/lib/seeder.js`): - Uma entrada recusada antes (`quota`, `header-invalid`) é tentada de novo uma vez a cada corte menor. - Escritor sem caixa num grupo privado: `header-invalid` enquanto contado, toco depois de liberado (`groupVerdict(..., released)`, na admissão, na partida, no portão e na reverificação de caixa). Se a liberação some, é julgado de novo e sai. -Documentos: `docs/protocol.md` (bit `CAP_WRITER_CUTS`, entrada de `buildGroup` e `buildCommunity` com `signed`, "Slot walk" e "Writers named by a node" sobre o sem caixa, regra do `auth.seq` em "Diary cuts on own writers" e em "Community distributors", parágrafo novo "Sparse copies", "Cut writers" do semeador reescrito). `docs/rpc.md` não muda: nenhum método nem resultado mudou. +Metas de conclusão com cópia esparsa (segunda rodada, pontos (a) e (b)): + +- `packages/core/lib/sparse-held.js` (novo, roda no Node, no Bare e no navegador): `advertisedHeld(core, end)` é verdadeiro quando algum par conectado já sincronizou, todo par sincronizado já respondeu pelos blocos abaixo de `end` e todo bloco abaixo de `end` que um deles anuncia está guardado. Depois do comprimento contíguo de um par, o bitfield dele só chega como resposta ao pedido de um intervalo de download, então o par conta como respondido quando chegou uma página do bitfield que cobre o intervalo. +- `diaryWriterCut(entries, G, W)` em `group.js`: o corte que a caminhada do diário dá ao escritor (0 quando liberado, `null` sem corte), igual ao `groupWalkCut` do semeador. +- Nó core (`sync.js`): na meta de `--once`, um escritor alheio guardado até o anunciado e com buraco depois conta como completo quando `advertisedHeld` vale até `min(comprimento assinado, MAX_LOG_BLOCKS)` (`sparseMet`, conferido de novo a cada `SPARSE_CHECK_MS` = 100 ms enquanto a meta espera). O próprio escritor cortado pelo diário conta como enviado quando o semeador o guarda até `max(corte, 1)`, nunca além do comprimento local; os cortes são lidos do diário quando a sincronização abre os escritores. +- Nó do navegador (`web-member/lib/sync.js`): as mesmas duas regras (`sparseMet` em `unmetGroupGoals`; `pushTarget` em `checkMedia`), com os cortes relidos a cada varredura do diário (`fold.cutEntries`, `ownCutEntries`), e `checkMedia` roda de novo depois de cada varredura. +- Serviço de contas (`pages-groups.js`): o `fetch` de um escritor listado termina, antes de `FETCH_TIMEOUT_MS`, quando todo bloco que o semeador anuncia abaixo do comprimento assinado está guardado (conferido a cada 50 ms), e então a cópia conta como completa. Raízes e diários continuam com a regra do contíguo. + +Documentos: `docs/protocol.md` (bit `CAP_WRITER_CUTS`, entrada de `buildGroup` e `buildCommunity` com `signed`, "Slot walk" e "Writers named by a node" sobre o sem caixa, regra do `auth.seq` em "Diary cuts on own writers" e em "Community distributors", parágrafo novo "Sparse copies", "Cut writers" do semeador reescrito; na segunda rodada, "Completion goals" com a cópia esparsa, o próprio escritor cortado e o `fetch` do serviço de contas). `docs/rpc.md` não muda: nenhum método nem resultado mudou. ## Como verificar @@ -47,9 +55,28 @@ Cada arquivo sozinho, a partir da raiz, com `node `, e `npx standard` n | `apps/cli/test/pages-groups.test.js` | 3/3 | 153/153 | | `apps/cli/test/account-pages-groups.test.js` | 1/1 | 49/49 | +Segunda rodada (depois do merge da `main` com #710, #711, #713 e #714), cada arquivo sozinho: + +| Arquivo | Testes | Asserções | +|---|---|---| +| `packages/core/test/sync-groups-sparse.test.js` (asserções novas do `--once`) | 1/1 | 14/14 | +| `packages/web-member/test/sync-groups-sparse.test.js` (novo, DHT local) | 1/1 | 11/11 | +| `apps/cli/test/pages-groups-sparse.test.js` (novo, DHT local) | 1/1 | 8/8 | +| `packages/core/test/sync-groups-3.test.js` | 16/16 | 71/71 | +| `packages/core/test/sync-private.test.js` | 4/4 | 16/16 | +| `packages/core/test/group-member.test.js` | 13/13 | 151/151 | +| `packages/core/test/seeder-groups-cuts.test.js` | 14/14 | 327/327 | +| `packages/web-member/test/sync-groups-2.test.js` | 26/26 | 130/130 | +| `packages/web-member/test/private-ops.test.js` | 24/24 | 231/231 | +| `packages/web-member/test/group-memo.test.js` | 11/11 | 60/60 | +| `apps/cli/test/pages-groups.test.js` | 3/3 | 152/152 (153 na primeira rodada: a asserção sobre a Geral só existe quando a entrada na comunidade também entrou nela, `joined.group`; dá 152 também com o `fetch` novo tirado) | +| `apps/cli/test/account-pages-groups.test.js` | 1/1 | 49/49 | + +`npm run lint` passa. + O teste de ponta a ponta: A cria a comunidade e dois grupos; B escreve nos dois, congela o escritor de G1 no comprimento (dois posts e a saída depois do corte) e libera o de G2 (um post e a saída). Depois que o semeador guarda os dois esparsos, R, novo, sincroniza só com ele: guarda W1 abaixo do corte mais a saída e W2 como toco, e a comunidade e os dois grupos (info, membros e leitura) dobram igual a quando R recebe as cópias inteiras de B. -`apps/cli/lib/account/pages-groups.js` não tem teste próprio da leitura esparsa: usa o mesmo `readHeldBlocks` e a mesma dobra testados acima, e hoje o `fetch` dele marca o grupo como `incomplete` com uma cópia esparsa (ponto (b) abaixo), então a diferença ainda não aparece para quem usa. +Na segunda rodada ele confere também que o `sync --once` do dono (B) e o do leitor novo (R) terminam `ok` antes do prazo de 30 s. `packages/web-member/test/sync-groups-sparse.test.js` (novo, DHT local e semeador atrás de WebSocket) faz o mesmo no nó do navegador com um escritor congelado: o `--once` de B e o de R terminam `ok`, R guarda o escritor abaixo do corte mais a saída, a saída conta e o post depois do corte não. A saída ali é escrita direto no escritor, como faria um aparelho offline: `group.leave` também desliga a entrada no diário, e esse caso esbarra num problema anterior a este PR (ponto (g) abaixo). `apps/cli/test/pages-groups-sparse.test.js` (novo, arquivo próprio porque `pages-groups.test.js` já usa boa parte do tempo por arquivo do executor): com o semeador guardando esparso o escritor congelado de B, a Página entra na comunidade, entra no grupo e posta, e a leitura dela vem com `incomplete: false`, o post abaixo do corte e sem o de depois. Selar (`group-seal`, grupo privado) passa pela mesma conferência `loaded.partial` e não tem teste próprio aqui. Não rodados aqui, por regra do brief: a suíte inteira e a demo (a sessão que abriu esta roda a CI de entrega). @@ -87,6 +114,17 @@ Ponta a ponta (`packages/core/test/sync-groups-sparse.test.js`): - E1, `readBlocks` em `readCoreParts` (nó core lê os escritores como prefixo): 0/1, 11/12, falha "the sparse copies fold like the whole ones". O semeador e as cópias de R estão certos; só a dobra difere. - E2, M1 no semeador (escritores cortados guardados inteiros): tempo esgotado "run condition", o semeador nunca fica esparso. +Segunda rodada, uma mutação por vez (`node ` com a mudança tirada, depois restaurada): + +| Mutação | Arquivo | Resultado | +|---|---|---| +| A1 meta do leitor sem `sparseMet` (nó core) | `packages/core/test/sync-groups-sparse.test.js` | 0/1, 13/14, falha "R's run ends ok before its deadline: tempo esgotado: :2/5, :1/4" | +| A2 meta do próprio escritor no comprimento inteiro (nó core) | o mesmo | 0/1, 13/14, falha "B's run ends ok before its deadline: tempo esgotado: :5/2, :4/1" | +| A3 meta do leitor sem `sparseMet` (navegador) | `packages/web-member/test/sync-groups-sparse.test.js` | 0/1, 10/11, falha "R's run ends ok before its deadline: tempo esgotado: :2/4" | +| A4 `pushTarget` devolve o comprimento inteiro (navegador) | o mesmo | 0/1, 10/11, falha "B's run ends ok before its deadline: tempo esgotado: :push 4" | +| B1 o `fetch` do escritor listado como antes | `apps/cli/test/pages-groups-sparse.test.js` | tempo esgotado "group join" (60 s de `indisponivel`) | +| B2 o `fetch` esparso sem `advertisedHeld` | o mesmo | tempo esgotado "group join" | + ## Decisões tomadas aqui - A regra do buraco mora na dobra (`bindCores`), não só nos leitores: assim o nó core, o nó do navegador e o serviço de contas dobram igual mesmo com cópias diferentes do mesmo escritor, e um bloco depois de um buraco abaixo do corte nunca conta. @@ -97,14 +135,19 @@ Ponta a ponta (`packages/core/test/sync-groups-sparse.test.js`): - Uma entrada recusada é tentada de novo uma vez a cada corte menor que a caminhada dá ao escritor: um corte faz caber na cota quem não cabia, e a liberação faz o sem caixa valer como toco. O custo é no máximo uma busca do bloco 0 por registro de corte. - Um escritor que sai por causa do que guarda fica com a entrada recusada (`groupRefused`), como uma recusa na admissão; sem isso ele seria admitido, buscado e derrubado a cada varredura. - A cota do que fica guardado usa `_overQuota` (núcleo inteiro) como atalho: só conta blocos e bytes quando o núcleo inteiro já passa da cota. Os bytes abaixo do corte são lidos um a um só nesse caso. +- Segunda rodada, (a): o anúncio que conta é o de todo par conectado que já sincronizou, não só o do semeador, e um par que ainda não respondeu pelo intervalo segura a meta. O próprio escritor usa o corte do diário (0 vira 1: o bloco 0 sempre fica), nunca o que o semeador lista. +- Segunda rodada, (b): só os escritores listados ganham a regra nova no serviço de contas; raiz e diário não são cortados e continuam pelo contíguo. O `fetch` ainda espera no máximo `FETCH_TIMEOUT_MS`; a diferença é terminar antes quando o anunciado já está guardado. +- (c) As dobras de catálogo (comunidade no índice) não recebem marcas e leem o prefixo, o que continua seguro. Mantido como está. +- (d) As cotas do lado do leitor usam o comprimento assinado, como já era. Mantido como está. +- (e) O semeador baixa os blocos depois do corte, até `MAX_LOG_BLOCKS`, só para ler o tipo; os pedaços de 256 limitam o que fica guardado além do mantido, não o que trafega. Mantido como está. - O fragmento de changelog de `feat/writer-slot-walk` (que diz que o semeador guarda o escritor inteiro) não foi mexido: é de outro PR; o deste PR diz o comportamento novo. ## Riscos residuais e pontos em aberto -- (a) Meta de `sync --once` (nó core e nó do navegador): `groupUnmet` quer `max(anunciado, core.length)` blocos contíguos de cada escritor alheio, e o dono de um escritor cortado espera o semeador listar o comprimento inteiro (`pushed`). Com o semeador esparso, nenhum dos dois chega lá, e um `--once` num grupo com escritor cortado só termina pelo tempo limite. Padrão recomendado: dar o escritor por completo quando o contíguo alcança o anunciado e todo bloco que um par conectado anuncia (bitfield) abaixo do comprimento assinado está guardado; para o próprio escritor, comparar com o corte do diário. Não feito aqui: está fora do escopo do brief e mexe nas duas sincronizações. -- (b) Serviço de contas: `fetch` em `pages-groups.js` dá uma cópia por completa só com o contíguo no comprimento, então um grupo com escritor cortado esparso fica `incomplete` e entrar, postar ou selar pela Página responde `indisponivel`. Padrão recomendado: completa quando todo bloco que o semeador anuncia está guardado. -- (c) As dobras de catálogo (comunidade no índice) não recebem marcas; leem o prefixo, o que continua seguro. Sem mudança. -- (d) As cotas do lado do leitor usam o comprimento assinado (já era assim). -- (e) Banda do semeador: os blocos depois do corte, até `MAX_LOG_BLOCKS`, são baixados só para ler o tipo. Os pedaços de 256 limitam o que fica guardado além do mantido, não o que trafega. -- (f) Item 4 do lado do leitor: as sincronizações (core e navegador) ainda recusam com `header-invalid` o cabeçalho sem caixa num grupo privado antes de guardá-lo, então um leitor que nunca o teve continua com a dobra `incomplete`, embora o semeador agora mantenha o bloco 0. Padrão recomendado: guardar só o bloco 0 de um escritor sem caixa de grupo privado quando a caminhada do diário dele o libera, e deixar a dobra rejeitá-lo pelo cabeçalho. É uma mudança grande nas duas sincronizações; não feita aqui. -- O teste de ponta a ponta não confere o resultado `ok` do `sync --once` do leitor, por causa de (a). +- (a) e (b) feitos na segunda rodada (acima). +- (c), (d) e (e): decisões mantidas (acima). Riscos que ficam: (c) uma dobra de catálogo vê menos de um CW cortado do que a dobra completa (só o prefixo), nunca mais; (d) um escritor esparso perto da cota de um leitor conta pelo comprimento assinado, não pelo que o leitor guarda; (e) a banda do semeador para um escritor cortado longo continua a do escritor inteiro até `MAX_LOG_BLOCKS`. +- (f) Não feito: parado para decisão (pedido da rodada). As sincronizações (core e navegador) ainda recusam com `header-invalid` o cabeçalho sem caixa num grupo privado antes de guardá-lo, então um leitor que nunca o teve fica com a dobra `incomplete`, embora o semeador mantenha o toco. Fazer exige que o veredito do portão dependa da caminhada do diário que liga o escritor (o registro de corte 0, não só o bloco de entrada que o I1 admite), levantar a recusa da rodada (`refused`) quando essa caminhada muda, guardar só `[0, 1)`, pular esse escritor em `recheckGroupBoxes`, na revalidação da partida e na meta de conclusão, e decidir o que acontece quando a liberação some (registro de corte apagado). Isso muda as regras de admissão (I1 a I4) nas duas sincronizações. +- (g) Encontrado na segunda rodada, anterior a este PR (vem da #713): o semeador lista um escritor que o diário desligou com `group.leave` com o seq desse `group-join` `on: false`, e o portão do leitor exige `on: true` naquele seq, então recusa como `unbound` (no navegador, visto no teste: `group-skipped` com `reason: 'unbound'`; no nó core pela leitura do código, `joinVerdict`), cobra como dica ruim do semeador e nunca guarda o escritor. Com o diário guardado, a caminhada espera o escritor e a dobra pode ficar `incomplete`. Não mexido aqui. +- O "respondeu" de `advertisedHeld` é uma aproximação: uma página de bitfield criada por outra mensagem (um `range` ou um `bitfield` do par) antes da resposta ao pedido faz o par contar como respondido cedo. O efeito é um `--once` terminar antes de um bloco anunciado chegar: a dobra trata a cópia como o prefixo (o que está depois do buraco e não chegou não conta) até a próxima sincronização, sem erro. +- No nó core, os cortes do próprio diário são lidos uma vez, quando a sincronização abre os escritores: um corte escrito durante a mesma execução só vale na próxima. +- O dono dá o escritor por enviado no corte; as saídas e os blocos de chave depois do corte podem chegar ao semeador só depois que o `--once` do dono terminou (o semeador os busca quando o dono aparece de novo). From ea6088342f7fed3043b9dfbf7bf95cd3ed85ab85 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 02:27:56 -0300 Subject: [PATCH 12/25] docs(seeder): name a writer cut at 0 once the releases are spent as a stub groupWalkCut gives 0 for a released writer and for one cut at 0 while still counted, and both are kept as stubs; the comments named only the first. --- packages/core/lib/seeder.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/core/lib/seeder.js b/packages/core/lib/seeder.js index 254cf7c2..fbbdd776 100644 --- a/packages/core/lib/seeder.js +++ b/packages/core/lib/seeder.js @@ -101,8 +101,9 @@ // cut and, past it, only its group-leave and group-key blocks, which the // fold still counts and readers read past a hole (_cutGroupCore): the rest // is fetched, read and cleared, and the quota counts what is kept. A writer -// cut at 0 (released) is such a stub: block 0, its leaves and its key -// blocks, also a replaced boxless writer of a private group, which readers +// cut at 0 (released, or cut at 0 while counted once the releases are +// spent) is such a stub: block 0, its leaves and its key blocks, also a +// replaced boxless writer of a private group, which readers // then reject by its header. Purges happen for a dropped or blocked diary, // a root its blocked or revoked creator bound, a root key in the blocklist snapshot, a fork or the // quota. @@ -819,9 +820,11 @@ async function localMediaVerdict (core, diaryHex) { // public one, else 'header-invalid' as in the fold and member sync. A // version 1 header (the validators' 'legacy') is 'header-invalid' too: no // wire reason of its own, and the startup re-gate purges it. `released`: the -// writer's diary cut it at 0, so a boxless writer of a private group (one -// its diary replaced) is kept as a stub, which readers fetch and reject by -// its header instead of waiting for it (#350). +// writer is kept as a stub, cut at 0 (groupWalkCut: released by its diary, +// or cut at 0 while still counted once the releases are spent), so a +// boxless writer of a private group (one its diary replaced) stays as a +// stub, which readers fetch and reject by its header instead of waiting +// for it (#350). function groupVerdict (raw, key, group, diary, visibility = null, released = false) { try { const header = decodeBlock(raw) @@ -4285,8 +4288,8 @@ class Seeder { entry.keptPastBytes -= bytes } this._rearmGroupDownload(entry) - // A boxless writer kept as a stub is judged again once no longer - // released. + // A boxless writer kept as a stub is judged again once its cut is no + // longer 0. if (old === 0 && cut !== 0) { const visibility = this._groupVisibility(entry.group) if (visibility !== null) this._recheckGroupBoxes(entry.group, visibility) From 00bbc48eae2ba4233a4608b58cb1320ca2189fe5 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 02:30:50 -0300 Subject: [PATCH 13/25] fix(sparse): count a peer answered by its bitfield segment, skip holes by bits A want asks for a whole bitfield segment and the answer carries its first page and every later page the peer has, so a page it holds nothing in never comes: a sparse peer on a log past 32768 blocks never counted as answered. readHeldBlocks now jumps past a hole to the next held block through the bitfield has() reads, instead of one has() per block. --- packages/core/lib/sparse-held.js | 75 +++++++++++++++------- packages/core/lib/store.js | 4 ++ packages/core/test/sparse-held.test.js | 87 ++++++++++++++++++++++++++ packages/web-member/lib/node-store.js | 4 ++ 4 files changed, 149 insertions(+), 21 deletions(-) create mode 100644 packages/core/test/sparse-held.test.js diff --git a/packages/core/lib/sparse-held.js b/packages/core/lib/sparse-held.js index 66fb2c4f..0db08b02 100644 --- a/packages/core/lib/sparse-held.js +++ b/packages/core/lib/sparse-held.js @@ -7,24 +7,30 @@ // Hypercore (10 and 11) keeps what a peer advertises per replicator Peer // (core.peers, as deletion.js advertisedBy reads it): its contiguous range, // then its bitfield. Past that range the bitfield only arrives as the answer -// to the want a download range sends, so until a page of it is here a block -// the peer holds there reads as not advertised: such a peer has not -// answered yet, and a goal that trusted it would end before the block came. +// to the want a download range sends: one want per bitfield segment, which +// the peer answers with one message from the segment's first block up to +// the last page it has in it. Until that answer is here a block the peer +// holds there reads as not advertised: such a peer has not answered yet, +// and a goal that trusted it would end before the block came. // Browser-safe: no Node built-ins. -// RemoteBitfield's page size, the same in Hypercore 10 and 11. -const BITFIELD_PAGE = 32768 +// RemoteBitfield's segment size, the same in Hypercore 10 and 11: a want +// asks for one segment. +const BITFIELD_SEGMENT = 2097152 -// Whether `peer` has told this core what it holds of [its contiguous -// length, end): it has synced, and either holds nothing past its contiguous -// range below end or a page of its bitfield covering that range arrived. -function peerAnswered (peer, end) { +// Whether `peer` has told this core what it holds of [from, end) past its +// contiguous range: it has synced, and either holds nothing past that range +// below end or the answer for each bitfield segment of it arrived. That +// answer always carries the segment's first page; a later page the peer +// holds nothing in is never sent, so its absence once the first page is +// here means nothing held there, not a missing answer. +function peerAnswered (peer, end, from = 0) { if (peer.remoteSynced !== true) return false - const from = peer.remoteContiguousLength + const start = Math.max(from, peer.remoteContiguousLength) const to = Math.min(end, peer.remoteLength || 0) - if (from >= to) return true - for (let p = Math.floor(from / BITFIELD_PAGE); p * BITFIELD_PAGE < to; p++) { - if (peer.remoteBitfield.getBitfield(p * BITFIELD_PAGE) === null) return false + if (start >= to) return true + for (let s = Math.floor(start / BITFIELD_SEGMENT); s * BITFIELD_SEGMENT < to; s++) { + if (peer.remoteBitfield.getBitfield(s * BITFIELD_SEGMENT) === null) return false } return true } @@ -42,17 +48,17 @@ function nextAdvertised (peer, from, end) { } // -> true once some connected peer of `core` has synced, every synced peer -// has answered for the blocks below `end` (peerAnswered) and every block -// below `end` one of them advertises is held. A peer that has not synced -// yet is not waited for. Never throws (false on any error). -async function advertisedHeld (core, end) { +// has answered for the blocks of [from, end) (peerAnswered) and every block +// of it one of them advertises is held. A peer that has not synced yet is +// not waited for. Never throws (false on any error). +async function advertisedHeld (core, end, from = 0) { try { const peers = [...core.peers].filter((p) => p.remoteSynced === true) if (peers.length === 0) return false - for (const peer of peers) if (!peerAnswered(peer, end)) return false - const from = core.contiguousLength + for (const peer of peers) if (!peerAnswered(peer, end, from)) return false + const start = Math.max(from, core.contiguousLength) for (const peer of peers) { - let i = nextAdvertised(peer, from, end) + let i = nextAdvertised(peer, start, end) while (i !== -1) { if (!(await core.has(i))) return false i = nextAdvertised(peer, i + 1, end) @@ -64,4 +70,31 @@ async function advertisedHeld (core, end) { } } -module.exports = { advertisedHeld, peerAnswered, BITFIELD_PAGE } +// What `core` holds, as Hypercore 10 and 11 keep it: core.core.bitfield, the +// structure has() itself reads for an open default session. null for any +// other session (a snapshot, a batch), which then asks has() per block. +function heldBits (core) { + try { + if (core.opened !== true) return null + if (core.state && typeof core.state.isDefault === 'function' && !core.state.isDefault()) return null + const bits = core.core && core.core.bitfield + return bits && typeof bits.firstSet === 'function' ? bits : null + } catch { + return null + } +} + +// The first block of [from, end) `core` holds, or -1: a sparse copy past +// its hole is mostly empty, and a scan of has() per block would cost its +// signed length. +async function nextHeld (core, from, end) { + const bits = heldBits(core) + if (bits !== null) { + const i = bits.firstSet(from) + return i >= 0 && i < end ? i : -1 + } + for (let i = from; i < end; i++) if (await core.has(i)) return i + return -1 +} + +module.exports = { advertisedHeld, peerAnswered, nextHeld, BITFIELD_SEGMENT } diff --git a/packages/core/lib/store.js b/packages/core/lib/store.js index f3f11107..143ebea0 100644 --- a/packages/core/lib/store.js +++ b/packages/core/lib/store.js @@ -9,6 +9,7 @@ const sodium = require('sodium-universal') const Corestore = require('corestore') const { validateRecord, decodeBlock, MAX_BLOCK_BYTES, MAX_LOG_BLOCKS, MAX_LOG_BYTES } = require('./records') const { presentState } = require('./deletion') +const { nextHeld } = require('./sparse-held') // Private directory: created 0o700 and tightened to 0o700 when it exists. // Follows symlinks: the data dir itself may be a symlink the user chose. @@ -532,6 +533,9 @@ async function readHeldBlocks (core) { const block = (await core.has(i)) ? await core.get(i, { wait: false, raw: true }) : null if (block === null) { hole = true + const next = await nextHeld(core, i + 1, limit) + if (next === -1) break + i = next - 1 continue } const record = decodeBlock(block) diff --git a/packages/core/test/sparse-held.test.js b/packages/core/test/sparse-held.test.js new file mode 100644 index 00000000..f9751da1 --- /dev/null +++ b/packages/core/test/sparse-held.test.js @@ -0,0 +1,87 @@ +'use strict' + +// Sparse copies against bare Hypercores (#350, lib/sparse-held.js): a peer +// that holds a few blocks of a long log answers for all of it, and a held +// block past a hole is found without asking has() for every block. + +const test = require('brittle') +const b4a = require('b4a') +const Hypercore = require('hypercore') +const path = require('node:path') +const { tmpDir } = require('./helpers/tmp') +const { advertisedHeld, nextHeld } = require('../lib/sparse-held') +const { readHeldBlocks } = require('../lib/store') + +function link (a, b) { + const left = a.replicate(true) + const right = b.replicate(false) + left.pipe(right).pipe(left) + return () => { + left.destroy() + right.destroy() + } +} + +async function open (t, dir, name, key) { + const core = new Hypercore(path.join(dir, name), key) + await core.ready() + t.teardown(() => core.close(), { force: true }) + return core +} + +// W: a log of `length` blocks, `leaveAt` a group-leave, the rest posts. +// S holds [0, 10) and the leave, fetched from W, which then goes. +async function sparseSource (t, length, leaveAt) { + const dir = tmpDir(t) + const w = await open(t, dir, 'w') + const blocks = [] + for (let i = 0; i < length; i++) { + blocks.push(b4a.from(JSON.stringify(i === leaveAt ? { type: 'group-leave' } : { type: 'group-post', text: 'p' + i }))) + } + for (let i = 0; i < length; i += 5000) await w.append(blocks.slice(i, i + 5000)) + const s = await open(t, dir, 's', w.key) + const unlink = link(w, s) + await s.update({ wait: true }) + await s.download({ start: 0, end: 10 }).done() + await s.download({ start: leaveAt, end: leaveAt + 1 }).done() + unlink() + return { dir, key: w.key, s } +} + +test('a peer holding a prefix and one block of a long log answers for all of it (#350)', async (t) => { + t.timeout(60000) + // The leave sits in the first bitfield page; the signed length reaches a + // second page the peer holds nothing in, so it never sends that page. + const { dir, key, s } = await sparseSource(t, 40000, 500) + const r = await open(t, dir, 'r', key) + link(s, r) + await r.update({ wait: true }) + r.download({ start: 0, end: r.length }) + const deadline = Date.now() + 8000 + let held = false + while (!held && Date.now() < deadline) { + held = await advertisedHeld(r, r.length) + if (!held) await new Promise((resolve) => setTimeout(resolve, 100)) + } + t.ok(held, 'every advertised block is held well before the deadline') + t.is(r.contiguousLength, 10) + t.ok(await r.has(500), 'the leave came') + t.absent(await r.has(10), 'nothing the peer does not hold') +}) + +test('a held block past a hole is found without asking has() for every block (#350)', async (t) => { + t.timeout(60000) + const { s } = await sparseSource(t, 40000, 35000) + const real = s.has.bind(s) + let calls = 0 + s.has = (...args) => { + calls++ + return real(...args) + } + t.is(await nextHeld(s, 10, s.length), 35000, 'the next held block') + t.is(await nextHeld(s, 35001, s.length), -1, 'none after it') + calls = 0 + const entries = await readHeldBlocks(s) + t.alike(entries.map((e) => e.seq), [...Array(10).keys(), 35000], 'the prefix and the leave past the hole') + t.ok(calls < 100, 'has() asked ' + calls + ' times, not once per block') +}) diff --git a/packages/web-member/lib/node-store.js b/packages/web-member/lib/node-store.js index dc94b762..4d286b03 100644 --- a/packages/web-member/lib/node-store.js +++ b/packages/web-member/lib/node-store.js @@ -50,6 +50,7 @@ const { const { mediaHeader } = require('@varanda/core/lib/media') const { validateGroupRecord, validateGroupHeader, validateWriterHeader, MAX_GROUPS_PER_DIARY, MAX_GROUP_WRITERS } = require('@varanda/core/lib/group') const { presentState, clearRanges } = require('@varanda/core/lib/deletion') +const { nextHeld } = require('@varanda/core/lib/sparse-held') const { MAX_INVENTORY } = require('@varanda/core/lib/admission') const { SEEN_BLOB_BYTES, SEEN_BLOB_B64 } = require('@varanda/core/lib/seen-blob') const { normalizeBlocks } = require('@varanda/core/lib/member-block') @@ -623,6 +624,9 @@ async function readHeldBlocks (core) { const block = (await core.has(i)) ? await core.get(i, { wait: false, raw: true }) : null if (block === null) { hole = true + const next = await nextHeld(core, i + 1, limit) + if (next === -1) break + i = next - 1 continue } const record = decodeBlock(block) From af985d3c5157a6bc3ebf1fa3603db495abc3628c Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 02:50:18 -0300 Subject: [PATCH 14/25] fix(seeder): end a chunk past a cut fed by sparse peers and read it again A chunk past a cut waited for every one of its blocks, so a seeder fed only by peers that keep the writer sparse never left the first chunk and never fetched a leave or key block beyond it. A chunk now ends once every block its synced peers advertise is held; what it holds is kept or cleared as before, and a chunk read that way joins the binding's unsure range (unsureFrom, unsureTo), read again when a peer joins and at growing intervals until a peer serves it whole. --- packages/core/lib/seeder.js | 194 +++++++++++++++--- packages/core/test/helpers/seeder-groups.js | 2 +- packages/core/test/seeder-groups-cuts.test.js | 67 +++++- 3 files changed, 233 insertions(+), 30 deletions(-) diff --git a/packages/core/lib/seeder.js b/packages/core/lib/seeder.js index fbbdd776..1d73820c 100644 --- a/packages/core/lib/seeder.js +++ b/packages/core/lib/seeder.js @@ -229,6 +229,7 @@ const { const { validateMediaHeader, latestMediaCores } = require('./media') const { validateGroupHeader, validateWriterHeader, validateGroupRecord, groupJoins, slotWalkState, slotWalkStep } = require('./group') const { createGroupIndex } = require('./group-index') +const { advertisedHeld } = require('./sparse-held') const { attachGroups } = require('./groups-net') const { presentState, presentLength, clearDeleted, downloadExcept, downloadExceptPages, pageTargets, deletedRanges, shownMedia } = require('./deletion') const { createEngagementIndex } = require('./engagement-index') @@ -647,7 +648,7 @@ function mediaBinding (value) { // creator left for another reason (quota, its own drop). function groupBinding (value) { if (!isPlainObject(value)) return null - const { diary, group, seq, by, bound, issuer, cut, clearedFrom, clearedTo } = value + const { diary, group, seq, by, bound, issuer, cut, clearedFrom, clearedTo, unsureFrom, unsureTo } = value if (typeof diary !== 'string' || !HEX64.test(diary)) return null if (typeof group !== 'string' || !HEX64.test(group)) return null if (!Number.isSafeInteger(seq) || seq < 1) return null @@ -657,14 +658,19 @@ function groupBinding (value) { if (out.bound === true && typeof issuer === 'string' && HEX64.test(issuer)) out.issuer = issuer // A writer its diary cut is kept below `cut` and, in [clearedFrom, // clearedTo), only its leaves and key blocks (_cutGroupCore); cut 0 is a - // released writer's stub. A range that does not fit the cut reads as - // nothing cleared yet. + // stub. A range that does not fit the cut reads as nothing cleared yet. + // [unsureFrom, unsureTo) inside it was read from sparse peers only and is + // read again; one that does not fit reads as read whole. if (Number.isSafeInteger(cut) && cut >= 0) { const below = keptBelow(cut) const fits = Number.isSafeInteger(clearedFrom) && Number.isSafeInteger(clearedTo) && below <= clearedFrom && clearedFrom <= clearedTo out.cut = cut out.clearedFrom = fits ? clearedFrom : below out.clearedTo = fits ? clearedTo : below + if (fits && Number.isSafeInteger(unsureFrom) && Number.isSafeInteger(unsureTo) && clearedFrom <= unsureFrom && unsureFrom < unsureTo && unsureTo <= clearedTo) { + out.unsureFrom = unsureFrom + out.unsureTo = unsureTo + } } return out } @@ -979,6 +985,24 @@ function keptPastCut (raw) { // Blocks past a cut read per step of _cutGroupCore: what a seeder stores // beyond what it keeps stays within one chunk. const CUT_CHUNK = 256 +// A chunk past a cut whose fetch has not ended is checked against what its +// peers advertise (_groupPastRange): soon, then less often while a peer is +// connected; a peer joining checks it again at once. +const CUT_POLL_MIN_MS = 100 +const CUT_POLL_MAX_MS = 2000 +// A range past a cut that sparse peers only fed (binding unsureFrom, +// unsureTo) is read again (_cutGroupCore): REREAD_PEER_MS after a peer +// joins, so that it has synced, and at intervals growing from +// REREAD_MIN_MS to REREAD_MAX_MS, until a pass reads each chunk whole. +const REREAD_PEER_MS = 1000 +const REREAD_MIN_MS = 1000 +const REREAD_MAX_MS = 5 * 60 * 1000 + +// Widens range { from, to } (null: none) to cover [from, to). +function widen (range, from, to) { + if (range === null) return { from, to } + return { from: Math.min(range.from, from), to: Math.max(range.to, to) } +} // The held blocks of [from, to) a cut keeps, as [seq, bytes], and the runs // [start, end) of the others, to clear. @@ -4264,6 +4288,8 @@ class Seeder { delete b.cut delete b.clearedFrom delete b.clearedTo + delete b.unsureFrom + delete b.unsureTo } else { const below = keptBelow(cut) if (old === null) { @@ -4273,6 +4299,13 @@ class Seeder { b.clearedFrom = Math.max(b.clearedFrom, below) b.clearedTo = Math.max(b.clearedTo, b.clearedFrom) } + if (Number.isSafeInteger(b.unsureFrom)) { + b.unsureFrom = Math.max(b.unsureFrom, b.clearedFrom) + if (b.unsureFrom >= b.unsureTo) { + delete b.unsureFrom + delete b.unsureTo + } + } b.cut = cut } this._persist().catch(noop) @@ -4280,6 +4313,8 @@ class Seeder { if (!entry || entry.dropped) return entry.cut = cut entry.cutConfirmed = true + entry.rereadAt = null + entry.rereadLeft = null entry.belowBytes = 0 entry.belowCounted = 0 for (const [i, bytes] of entry.keptPast) { @@ -4666,7 +4701,10 @@ class Seeder { // the cut counted so far (`belowCounted` of them), `counted` once the // range read was read again in this serve, `past` the fetch of blocks // past the cut being read, `cutting`, `cutAgain` and `wake` one pass at - // a time. + // a time. A read again of the binding's unsure range (`reread` asked, + // `rereadAt` the next chunk, `rereadLeft` the chunks still read from + // sparse peers only in this pass) runs at `rereadTimer`, backing off + // by `rereadDelay`. cut, cutConfirmed: !restored, keptPast: new Map(), @@ -4677,7 +4715,12 @@ class Seeder { past: null, cutting: false, cutAgain: false, - wake: null + wake: null, + reread: false, + rereadAt: null, + rereadLeft: null, + rereadTimer: null, + rereadDelay: REREAD_MIN_MS } const checkQuota = () => { if (entry.dropped || this._closing) return @@ -4689,14 +4732,24 @@ class Seeder { const onDownload = (index) => { if (entry.cut !== null && index < keptBelow(entry.cut) && core.byteLength > this.maxLogBytes) this._cutGroupCore(key, entry) } + // A peer that joins may hold what sparse peers did not. + const onPeer = () => { + if (entry.dropped || this._closing) return + entry.rereadDelay = REREAD_MIN_MS + this._armReread(key, entry, REREAD_PEER_MS) + } core.on('append', checkQuota) core.on('download', onDownload) + core.on('peer-add', onPeer) entry.stopQuota = () => { core.off('append', checkQuota) core.off('download', onDownload) + core.off('peer-add', onPeer) try { if (entry.past) entry.past.range.destroy() } catch {} + if (entry.rereadTimer !== null) clearTimeout(entry.rereadTimer) + entry.rereadTimer = null if (entry.wake !== null) entry.wake() } entry.stopFork = watchFork(core, (reason) => this._dropGroup(key, reason)) @@ -4708,6 +4761,7 @@ class Seeder { this._groupServed.set(key, entry) checkQuota() if (entry.dropped) return + this._armReread(key, entry, REREAD_MIN_MS) this._scanRoot(key, entry) // A peer that served block 0 to the quarantine and then withholds it // from the main store must not keep the group's answers incomplete @@ -4747,7 +4801,10 @@ class Seeder { } // Waits for blocks [from, to) of a cut writer, fetched by one range kept - // across passes: true once all are local, false when another pass was + // across passes: 'done' once all are local; 'sparse' once every block of + // it a connected peer advertises is (sparse-held.js advertisedHeld), as + // when its only sources keep it sparse themselves (another seeder past + // the same cut), whose fetch would never end; null when another pass was // asked for first or the fetch ended without them. async _groupPastRange (entry, from, to) { let p = entry.past @@ -4764,16 +4821,57 @@ class Seeder { entry.past = q p = q } - if (!p.done && !p.failed) { - await new Promise((resolve) => { - entry.wake = resolve - p.settled.then(resolve) + const core = entry.core + let delay = CUT_POLL_MIN_MS + let held = null + while (!p.done && !p.failed) { + if (await advertisedHeld(core, to, from)) { + held = (await core.has(from, to)) ? 'done' : 'sparse' + break + } + if (p.done || p.failed) break + let timer = null + let onPeer = null + const why = await new Promise((resolve) => { + entry.wake = () => resolve('wake') + p.settled.then(() => resolve('settled')) + onPeer = () => resolve('peer') + core.on('peer-add', onPeer) + if (core.peers.length > 0) timer = setTimeout(() => resolve('tick'), delay) }) entry.wake = null + core.off('peer-add', onPeer) + if (timer !== null) clearTimeout(timer) + if (why === 'wake') break + delay = why === 'tick' ? Math.min(delay * 2, CUT_POLL_MAX_MS) : CUT_POLL_MIN_MS + } + if (p.done) held = 'done' + if (held === null) return null + // Its blocks no peer advertised would otherwise come later, past the + // read, and stay. + if (!p.done) { + try { + p.range.destroy() + } catch {} } - if (!p.done) return false if (entry.past === p) entry.past = null - return true + return held + } + + // Arms the next read again of writer K's unsure range in `delay` ms (none + // without one), replacing any armed before. + _armReread (key, entry, delay) { + if (entry.rereadTimer !== null) clearTimeout(entry.rereadTimer) + entry.rereadTimer = null + const b = this._groups.get(key) + if (entry.dropped || this._closing || !b || !Number.isSafeInteger(b.unsureFrom)) return + entry.rereadTimer = setTimeout(() => { + entry.rereadTimer = null + if (entry.dropped || this._closing || this._groupServed.get(key) !== entry) return + entry.reread = true + this._cutGroupCore(key, entry) + }, delay) + if (entry.rereadTimer.unref) entry.rereadTimer.unref() } // A writer its diary cut (docs/protocol.md section 10) keeps its blocks @@ -4784,8 +4882,12 @@ class Seeder { // they are one of those (Hypercore keeps the tree nodes the proofs of the // kept blocks need). A chunk is listed as read before its blocks go: after // a crash in between, the next serve reads the range again and clears - // what it still holds beyond the kept blocks. The quota counts what is - // kept. One pass at a time; growth or a new cut during a pass runs another. + // what it still holds beyond the kept blocks. A chunk read from sparse + // peers only (_groupPastRange 'sparse') joins the binding's unsure range, + // read again chunk by chunk once a peer joins and at growing intervals: + // a leave or key block those peers lacked comes from a fuller one, and a + // chunk read whole leaves the range. The quota counts what is kept. One + // pass at a time; growth or a new cut during a pass runs another. _cutGroupCore (key, entry) { if (entry.cutting) { entry.cutAgain = true @@ -4838,39 +4940,75 @@ class Seeder { } if (!entry.cutConfirmed) return // A lower cut first reads down from the range already read, so that - // range stays one piece; then up to the signed length. + // range stays one piece; then up to the signed length; then, when + // asked, the unsure range again. const end = Math.min(core.length, MAX_LOG_BLOCKS) const down = b.clearedFrom > below + const unsure = Number.isSafeInteger(b.unsureFrom) let from let to + let again = false if (down) { from = Math.max(below, b.clearedFrom - CUT_CHUNK) to = b.clearedFrom } else if (b.clearedTo < end) { from = b.clearedTo to = Math.min(end, from + CUT_CHUNK) + } else if (entry.reread && unsure) { + again = true + from = entry.rereadAt === null ? b.unsureFrom : Math.max(entry.rereadAt, b.unsureFrom) + to = Math.min(b.unsureTo, from + CUT_CHUNK) } else { + entry.reread = false return } - const held = await this._groupPastRange(entry, from, to) + const res = from < to ? await this._groupPastRange(entry, from, to) : 'done' if (moved()) continue - if (!held) { + if (res === null) { if (entry.cutAgain) continue return } const { keep, clear } = await cutRange(core, from, to) if (moved()) continue - if (down) b.clearedFrom = from - else b.clearedTo = to - try { - await this._persist() - } catch (err) { - entry.counted = false - throw err - } - if (moved()) { - entry.counted = false - continue + if (again) { + if (res === 'sparse') entry.rereadLeft = widen(entry.rereadLeft, from, to) + entry.rereadAt = to + if (to >= b.unsureTo) { + const left = entry.rereadLeft + if (left === null) { + delete b.unsureFrom + delete b.unsureTo + } else { + b.unsureFrom = left.from + b.unsureTo = left.to + } + entry.reread = false + entry.rereadAt = null + entry.rereadLeft = null + entry.rereadDelay = Math.min(entry.rereadDelay * 2, REREAD_MAX_MS) + this._armReread(key, entry, entry.rereadDelay) + this._persist().catch(noop) + } + } else { + if (down) b.clearedFrom = from + else b.clearedTo = to + if (res === 'sparse') { + const u = widen(unsure ? { from: b.unsureFrom, to: b.unsureTo } : null, from, to) + b.unsureFrom = u.from + b.unsureTo = u.to + if (entry.rereadAt !== null) entry.rereadLeft = widen(entry.rereadLeft, from, to) + if (!unsure) this._armReread(key, entry, entry.rereadDelay) + } + try { + await this._persist() + } catch (err) { + entry.counted = false + throw err + } + if (moved()) { + entry.counted = false + continue + } } for (const [start, end] of clear) await core.clear(start, end) if (moved()) { diff --git a/packages/core/test/helpers/seeder-groups.js b/packages/core/test/helpers/seeder-groups.js index 0fe2005f..26b50131 100644 --- a/packages/core/test/helpers/seeder-groups.js +++ b/packages/core/test/helpers/seeder-groups.js @@ -127,7 +127,7 @@ async function member (t, bootstrap, assoc, fill, { certOf = (key) => validCert( }) await joinAndFlush(swarm, [topicFor(assoc.hex)]) const seederPeer = () => until(() => peers.find((p) => p.remoteRole === 'seeder'), { name: 'hello do semeador' }) - return { store, diary, key, cores, swarm, messages, seederPeer, certOf } + return { store, diary, key, cores, swarm, messages, peers, seederPeer, certOf } } async function admit (m, assoc) { diff --git a/packages/core/test/seeder-groups-cuts.test.js b/packages/core/test/seeder-groups-cuts.test.js index a533011c..9ff7b640 100644 --- a/packages/core/test/seeder-groups-cuts.test.js +++ b/packages/core/test/seeder-groups-cuts.test.js @@ -8,7 +8,8 @@ const test = require('brittle') const b4a = require('b4a') -const { testnet, until } = require('./helpers/net') +const { testnet, until, localSwarmOptions } = require('./helpers/net') +const { createSwarm, attachAdmission } = require('../lib/net') const { liveGroupJoins, groupWalkStep, groupWalkCut } = require('../lib/seeder') const { groupJoinRecord, @@ -571,3 +572,67 @@ test('a boxless writer of a private group its diary released is kept as a stub; t.absent(second.events.some((e) => e.event === 'group-rejected' && e.key === N), 'not purged') t.is(first.events.filter((e) => e.event === 'fatal').length + second.events.filter((e) => e.event === 'fatal').length, 0) }) + +test('a seeder fed only by a seeder that keeps a writer sparse reads past cut + 256, then reads it again once a fuller peer joins (#350)', async (t) => { + t.timeout(240000) + const { bootstrap } = await testnet(t) + const assoc = makeAssociation() + const first = prepareSeederDir(t, assoc) + const one = await startSeeder(t, bootstrap, first.dataDir, { associationKey: assoc.hex }) + + // W: header, five posts, 300 more, a leave and one more post; cut at 3, + // so the leave sits past the first chunk read after the cut. + const { A, B, R, W } = await cutWorld(t, bootstrap, assoc) + const posts = [] + for (let i = 0; i < 300; i++) posts.push(b4a.from(JSON.stringify(groupPost({ text: 'mais ' + i })))) + await B.cores.w.append(posts) + const leaveAt = B.cores.w.length + await B.cores.w.append(b4a.from(JSON.stringify(groupLeave({ cuts: [{ writer: W, seq: leaveAt }] })))) + await B.cores.w.append(b4a.from(JSON.stringify(groupPost({ text: 'depois da saída' })))) + const length = B.cores.w.length + t.ok(leaveAt >= 3 + 256, 'the leave is past the first chunk') + await B.diary.append(groupCutRecord({ group: R, writer: W, cut: 3 })) + await admit(A, assoc) + await admit(B, assoc) + await admitted(one.events, W) + const e1 = one.seeder._groupServed.get(W) + await until(async () => e1.cut === 3 && !e1.cutting && one.seeder._groups.get(W).clearedTo === length && (await blocksHeld(e1.core, 0, length)).length === 4, { name: 'W esparso no primeiro semeador' }) + t.alike(await blocksHeld(e1.core, 0, length), [0, 1, 2, leaveAt], 'the first seeder keeps W below its cut and its leave') + t.absent('unsureFrom' in readGroups(first.dataDir)[W], 'read whole from its owner') + + // B goes: a second seeder reads W from the first only, which holds + // nothing of the first chunk past the cut. A carries B's diary to it (the + // gate fetches a diary from its announcer), never W. + const carried = A.store.get({ key: B.diary.key }) + await carried.ready() + await carried.download({ start: 0, end: B.diary.length }).done() + await B.swarm.destroy() + const second = prepareSeederDir(t, assoc) + const two = await startSeeder(t, bootstrap, second.dataDir, { associationKey: assoc.hex }) + const twoKey = hex(two.seeder.swarm.keyPair.publicKey) + const peer = await until(() => A.peers.find((p) => p.remoteRole === 'seeder' && p.peerKey === twoKey), { name: 'segundo semeador' }) + peer.send({ type: 'announce', key: A.key, cert: A.certOf(A.key) }) + peer.send({ type: 'announce', key: B.key, cert: B.certOf(B.key) }) + await admitted(two.events, W) + const e2 = two.seeder._groupServed.get(W) + await until(async () => !e2.cutting && two.seeder._groups.get(W).clearedTo === length && await e2.core.has(leaveAt), { name: 'W lido do primeiro semeador', timeoutMs: 60000 }) + t.alike(await blocksHeld(e2.core, 0, length), [0, 1, 2, leaveAt], 'the second seeder keeps the leave too') + t.is(e2.keptPast.size, 1, 'and counts it toward the quota') + const listed = readGroups(second.dataDir)[W] + t.alike([listed.unsureFrom, listed.unsureTo], [3, length], 'the range read from sparse peers only is listed as such') + + // B comes back with W whole: the range is read again, each chunk whole, + // and what is not kept is cleared again. + const back = createSwarm(localSwarmOptions(bootstrap)) + t.teardown(() => back.destroy(), { force: true }) + back.on('connection', (socket) => { + socket.on('error', () => {}) + B.store.replicate(socket) + attachAdmission(socket, { role: 'member', have: [] }) + }) + back.joinPeer(b4a.from(twoKey, 'hex')) + await until(() => !('unsureFrom' in two.seeder._groups.get(W)) && !e2.cutting, { name: 'W relido inteiro', timeoutMs: 60000 }) + t.alike(await blocksHeld(e2.core, 0, length), [0, 1, 2, leaveAt], 'what it read again past the cut is cleared again') + t.absent('unsureFrom' in readGroups(second.dataDir)[W], 'and the range is listed as read whole') + t.is(one.events.filter((e) => e.event === 'fatal').length + two.events.filter((e) => e.event === 'fatal').length, 0) +}) From f9c3b6cb71836f19040d7fb4387ce688ac865706 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 03:04:14 -0300 Subject: [PATCH 15/25] fix(sync): confirm a cut writer once the seeder holds its leaves past the cut The owner counted a writer its diary cut as pushed once a seeder held it up to the cut, so a leave or key block past the cut that never reached the seeder was confirmed and a once run ended ok with it on the device only. Both nodes now also wait until the seeder's replication peer advertises, or was served, each own group-leave and group-key block past the cut, and ask that peer for its bitfield there, since a full core sends no wants of its own. --- packages/core/lib/sparse-held.js | 54 ++++++++++++- packages/core/lib/sync.js | 78 +++++++++++++++++-- packages/core/test/sync-groups-sparse.test.js | 45 +++++++++++ packages/web-member/lib/sync.js | 50 +++++++++++- .../test/sync-groups-sparse.test.js | 51 ++++++++++++ 5 files changed, 269 insertions(+), 9 deletions(-) diff --git a/packages/core/lib/sparse-held.js b/packages/core/lib/sparse-held.js index 0db08b02..cc426739 100644 --- a/packages/core/lib/sparse-held.js +++ b/packages/core/lib/sparse-held.js @@ -12,8 +12,13 @@ // the last page it has in it. Until that answer is here a block the peer // holds there reads as not advertised: such a peer has not answered yet, // and a goal that trusted it would end before the block came. +// The owner of a cut writer reads the same per-peer state the other way: +// a seeder holds its leaves and key blocks past the cut once it advertises +// them or was served them (askHeld, peerHolds). // Browser-safe: no Node built-ins. +const { decodeBlock } = require('./records') + // RemoteBitfield's segment size, the same in Hypercore 10 and 11: a want // asks for one segment. const BITFIELD_SEGMENT = 2097152 @@ -97,4 +102,51 @@ async function nextHeld (core, from, end) { return -1 } -module.exports = { advertisedHeld, peerAnswered, nextHeld, BITFIELD_SEGMENT } +// Whether `peer` holds every block in `seqs`: it advertises it, or this +// core served it to that peer (`served`, a Set of indexes, or null). A +// peer never announces a block it gets to one that holds it already, so a +// full copy sees what it served this way only. +function peerHolds (peer, seqs, served = null) { + for (const seq of seqs) { + if (seq < peer.remoteContiguousLength) continue + if (peer.remoteBitfield.get(seq) !== true && (served === null || !served.has(seq))) return false + } + return true +} + +// A want handle that never resolves anything: the want stays for the +// peer's life, so blocks the peer gets there later are announced too. +const STANDING_WANT = { addWant () {}, removeWant () {} } + +// Asks `peer` for its bitfield segments holding `seqs`, once per segment. +// A full copy sends no wants of its own, and a peer tells a core what it +// holds past its contiguous range only in answer to one (Hypercore 11 also +// announces a block it gets only to peers that want its segment, or near +// the tip). Hypercore internals, as a download range calls them: 10's +// Peer#_maybeWant takes a range, 11's a want handle. +function askHeld (peer, seqs) { + if (typeof peer._maybeWant !== 'function') return + for (const seq of seqs) { + if (seq < peer.remoteContiguousLength) continue + try { + if (peer.wants && typeof peer.wants.add === 'function') peer._maybeWant(seq, STANDING_WANT) + else peer._maybeWant(seq, 1) + } catch {} + } +} + +// The blocks of [from, to) of a full local copy that a seeder keeps past a +// cut: its group-leave and group-key blocks (seeder.js keptPastCut). +async function keptPastSeqs (core, from, to) { + const out = [] + for (let seq = from; seq < to; seq++) { + let block = null + try { + block = decodeBlock(await core.get(seq, { wait: false, raw: true })) + } catch {} + if (block !== null && (block.type === 'group-leave' || block.type === 'group-key')) out.push(seq) + } + return out +} + +module.exports = { advertisedHeld, peerAnswered, nextHeld, peerHolds, askHeld, keptPastSeqs, BITFIELD_SEGMENT } diff --git a/packages/core/lib/sync.js b/packages/core/lib/sync.js index 210ea860..1da34c34 100644 --- a/packages/core/lib/sync.js +++ b/packages/core/lib/sync.js @@ -74,7 +74,7 @@ const { certForKey, seederCertForKey } = require('./cert') const { createBlocklistState, verifySnapshot, compareSnapshots } = require('./blocklist') const { attachBlocklist } = require('./blocklist-net') const { validateGroupHeader, validateWriterHeader, buildCommunity, groupJoinHistory, groupConfirmHistory, diaryWriterCut, MAX_GROUP_WRITERS, MAX_GROUPS_PER_DIARY, MAX_GROUP_WRITERS_PER_DIARY, MAX_GROUPS_PER_COMMUNITY } = require('./group') -const { advertisedHeld } = require('./sparse-held') +const { advertisedHeld, keptPastSeqs, askHeld, peerHolds } = require('./sparse-held') const { attachGroups, GROUPS_LIMIT_MAX, GROUPS_REQUEST_TIMEOUT_MS, MESSAGE_BURST: GROUPS_BURST, MESSAGE_RATE: GROUPS_RATE } = require('./groups-net') const { presentState, @@ -140,6 +140,9 @@ const GROUP_REQUEST_RATE = GROUPS_RATE / 2 // A foreign writer held past its prefix is checked again this often while // its goal waits on what its peers advertise (sparse-held.js). const SPARSE_CHECK_MS = 100 +// An own cut writer whose leaves or key blocks a seeder does not advertise +// yet is checked again this often (its answer to the want raises no event). +const KEPT_CHECK_MS = 250 // Page items whose diary or join seq a gate disproved, per seeder, before // this run stops paging that seeder. const MAX_BAD_HINTS = 32 @@ -2134,7 +2137,15 @@ async function runSync ({ continue } seen.add(g.group) - ownWriters.set(g.writer, { group: g.group, core, cut: null }) + // uploaded: replicator Peer -> the block indexes served to it. + const own = { group: g.group, core, cut: null, kept: null, keptReading: false, keptTimer: null, uploaded: new WeakMap() } + core.on('upload', (index, byteLength, from) => { + if (!from) return + let set = own.uploaded.get(from) + if (set === undefined) own.uploaded.set(from, (set = new Set())) + set.add(index) + }) + ownWriters.set(g.writer, own) groupList.push({ group: g.group, writer: g.writer, nonce: g.nonce }) groupSet.add(g.group) if (g.kind === 'community' || (g.writer === g.group && await ownRootKind(core) === 'community')) ownCommunities.add(g.group) @@ -2155,8 +2166,9 @@ async function runSync ({ } catch {} } } - // A writer the own diary cut is pushed whole at the cut: the seeders - // keep it sparse past it (docs/protocol.md "Sparse copies"). + // A writer the own diary cut is pushed whole up to the cut, and past it + // only its leaves and key blocks, all a seeder keeps there + // (docs/protocol.md "Sparse copies"). if (ownWriters.size > 0) { try { const entries = await readAll(diary) @@ -2633,9 +2645,14 @@ async function runSync ({ } const own = ownWriters.get(g.writer) // A writer its diary cut is kept to the cut, block 0 always: past it - // a seeder holds only what it keeps, never the whole length. + // a seeder holds only its leaves and key blocks, never the whole + // length, and those must be there too. const goal = own.cut === null ? own.core.length : Math.min(own.core.length, Math.max(own.cut, 1)) if (goal > pushed) unmet.push(g.writer + ':' + goal + '/' + pushed) + else if (goal < own.core.length) { + const missing = ownKeptMissing(own, seeders, g, goal) + if (missing !== -1) unmet.push(g.writer + ':corte ' + missing) + } } for (const [key, advertised] of groupTargets) { if (ownWriters.has(key)) continue @@ -2659,6 +2676,57 @@ async function runSync ({ return unmet } + // The own leaves and key blocks of cut writer `own` past its cut, read + // once per length (null while that read runs). + function ownKept (own) { + const length = Math.min(own.core.length, maxLogBlocks) + const k = own.kept + if (k !== null && k.from === own.cut && k.to === length) return k.seqs + if (own.keptReading || ended) return null + own.keptReading = true + const from = own.cut + const start = k !== null && k.from === from && k.to < length ? k.to : from + const before = start === from ? [] : k.seqs + track(keptPastSeqs(own.core, start, length), (seqs) => { + own.keptReading = false + own.kept = { from, to: length, seqs: before.concat(seqs) } + evaluate() + }, () => { + own.keptReading = false + }) + return null + } + + // The first own leave or key block of cut writer `own` that no verified + // seeder listing it at `goal` holds (-1 when one holds them all): past + // the cut a seeder keeps those blocks only, so the writer is pushed once + // one holds them. Each such seeder is asked for its bitfield there (a + // full core sends no wants of its own) and checked again soon. + function ownKeptMissing (own, seeders, g, goal) { + const seqs = ownKept(own) + if (seqs === null) return own.cut + if (seqs.length === 0) return -1 + let missing = seqs[0] + for (const p of seeders) { + const pass = p.groupPasses.get(g.group) + const it = pass && pass.items.get(g.writer) + if (!it || it.length < goal) continue + const peer = own.core.peers.find((x) => x.stream === p.socket) + if (!peer) continue + const served = own.uploaded.get(peer) || null + if (peerHolds(peer, seqs, served)) return -1 + askHeld(peer, seqs) + missing = seqs.find((seq) => !peerHolds(peer, [seq], served)) + } + if (own.keptTimer === null && !ended) { + own.keptTimer = later(() => { + own.keptTimer = null + evaluate() + }, KEPT_CHECK_MS) + } + return missing + } + // Whether foreign writer e, held up to a hole past what its seeders list, // holds every block a connected peer advertises below its signed length, // from the last check at its present length and prefix (sparse-held.js). diff --git a/packages/core/test/sync-groups-sparse.test.js b/packages/core/test/sync-groups-sparse.test.js index 79b5ab7f..45c78b5c 100644 --- a/packages/core/test/sync-groups-sparse.test.js +++ b/packages/core/test/sync-groups-sparse.test.js @@ -143,3 +143,48 @@ test('a fresh member fed only by a seeder folds a sparse frozen writer and a rel t.absent(full[g1].members.members.some((m) => m.member === b.key), 'control: the leave past the cut counts') t.alike(fromSeeder, full, 'the sparse copies fold like the whole ones') }) + +test('the owner of a cut writer counts it pushed only once the seeder holds its leave past the cut (#350)', async (t) => { + t.timeout(300000) + const { bootstrap } = await testnet(t) + const kp = generateAssociationKeyPair() + const seeder = await startSeeder(t, bootstrap, kp) + const a = await person(t, kp, 'Ana') + const b = await person(t, kp, 'Bia') + const { community } = await member.communityCreate(a, { name: 'Bairro', join: 'open' }) + const { group } = await member.groupCreate(a, { name: 'Horta', join: 'open', community }) + await exchange(a, b) + await member.communityJoin(b, { community }) + await member.groupJoin(b, { group }) + await member.groupPost(b, { group, text: 'antes' }) + const w = ownWriter(b, group) + const cut = (await writerEntries(b, group)).length + await offlineAppend(b, group, groupPost({ text: 'depois' })) + const leave = await offlineAppend(b, group, groupLeave({ seen: null, cuts: [{ writer: w, seq: cut + 1 }] })) + const s = await store.openStore(b.dataDir) + try { + await store.appendRecord(await store.openDiary(s), groupCutRecord({ group, writer: w, cut })) + } finally { + await s.close() + } + t.ok((await syncOnce(bootstrap, a)).result.ok, 'A pushes its diary and roots') + + // The seeder holds W below the cut but has not read past it yet. + const read = seeder._groupPastRange + let resume = null + const stalled = new Promise((resolve) => { resume = resolve }) + seeder._groupPastRange = async function (...args) { + await stalled + return read.apply(this, args) + } + const first = await syncOnce(bootstrap, b, { timeoutMs: 15000 }) + const errors = first.result.errors.join('; ') + t.absent(first.result.ok, 'B\'s run does not count W pushed at its cut alone') + t.ok(errors.includes(w + ':corte ' + leave.seq), 'it waits for the leave: ' + errors) + t.absent(await seeder._groupServed.get(w).core.has(leave.seq), 'control: the seeder lacks the leave') + + resume() + const second = await syncOnce(bootstrap, b, { timeoutMs: 30000 }) + t.ok(second.result.ok, 'once the seeder reads past the cut the run ends ok: ' + second.result.errors.join('; ')) + t.ok(await seeder._groupServed.get(w).core.has(leave.seq), 'with the leave at the seeder') +}) diff --git a/packages/web-member/lib/sync.js b/packages/web-member/lib/sync.js index f172d5cd..4c0330a4 100644 --- a/packages/web-member/lib/sync.js +++ b/packages/web-member/lib/sync.js @@ -186,6 +186,10 @@ // them on its quarantine channel and store none, and a once // run would end with the group behind. While a confirmation // is pending the pages are asked again (GROUP_PUSH_PAGE_MS). +// A writer the own diary cut counts up to the cut, and its +// own leaves and key blocks past it must be advertised by +// that peer, or served to it on the link, too +// (seederHoldsKept). // A seeder from before groups keeps the media rule. // groups the groups of kv 'groupWriters', plus each key of kv // 'groupReplicas' with nothing stored yet (a root asked for @@ -348,7 +352,7 @@ const { downloadExceptPages, pageTargets } = require('@varanda/core/lib/deletion') -const { advertisedHeld } = require('@varanda/core/lib/sparse-held') +const { advertisedHeld, keptPastSeqs, askHeld, peerHolds } = require('@varanda/core/lib/sparse-held') const { memoryStorage } = require('./storage') const { guardedWsStream } = require('./ws-stream') const store = require('./node-store') @@ -3369,7 +3373,7 @@ async function syncRun ({ // indexes a tombstone of the own diary deleted (media only; a group core // keeps it empty, seederHoldsMedia reads it for both). function pushEntry (key, core, event) { - const m = { key, core, pushed: -1, stop: null, uploaded: new WeakMap(), event, revoked: new Set() } + const m = { key, core, pushed: -1, stop: null, uploaded: new WeakMap(), event, revoked: new Set(), kept: null, keptReading: false } const onPeer = () => { checkMedia() pollMedia() @@ -3503,13 +3507,53 @@ async function syncRun ({ return cut === null ? length : Math.min(length, Math.max(cut, 1)) } + // The own leaves and key blocks of writer m past its cut, read once per + // length (null while that read runs; checkMedia runs again after it). + function ownKept (m, cut, length) { + const k = m.kept + if (k !== null && k.from === cut && k.to === length) return k.seqs + if (m.keptReading || ending) return null + m.keptReading = true + const start = k !== null && k.from === cut && k.to < length ? k.to : cut + const before = start === cut ? [] : k.seqs + track(keptPastSeqs(m.core, start, length), (seqs) => { + m.keptReading = false + m.kept = { from: cut, to: length, seqs: before.concat(seqs) } + checkMedia() + }, () => { + m.keptReading = false + }) + return null + } + + // Whether the seeder's peer on link l holds every own leave and key block + // of writer m past the cut the own diary gave it (advertised, or served + // on this link): past the cut a seeder keeps only those, so the writer is + // pushed once it holds them. The peer is asked for its bitfield there (a + // full core sends no wants of its own); a later check sees the answer. + function seederHoldsKept (m, l, length) { + if (m.key === m.group || ownCutEntries === null) return true + const cut = diaryWriterCut(ownCutEntries, m.group, m.key) + if (cut === null || cut >= length) return true + const seqs = ownKept(m, cut, Math.min(length, maxLogBlocks)) + if (seqs === null) return false + if (seqs.length === 0) return true + for (const peer of m.core.peers) { + if (peer.stream !== l.noise) continue + if (peerHolds(peer, seqs, m.uploaded.get(peer) || null)) return true + askHeld(peer, seqs) + return false + } + return false + } + function checkMedia () { const l = seederLink() if (l === null || ended) return for (const m of pushCores()) { const length = m.core.length if (length < 1 || m.pushed === length) continue - if (m.group ? seederHoldsGroup(m, l, pushTarget(m, length)) : seederHoldsMedia(m, l, length)) { + if (m.group ? seederHoldsGroup(m, l, pushTarget(m, length)) && seederHoldsKept(m, l, length) : seederHoldsMedia(m, l, length)) { m.pushed = length emit({ event: m.event, key: m.key, length }) } diff --git a/packages/web-member/test/sync-groups-sparse.test.js b/packages/web-member/test/sync-groups-sparse.test.js index 813f6173..38a4c3f8 100644 --- a/packages/web-member/test/sync-groups-sparse.test.js +++ b/packages/web-member/test/sync-groups-sparse.test.js @@ -112,3 +112,54 @@ test('a sparse cut writer: the owner\'s and a fresh reader\'s --once end ok (#35 t.ok(texts.includes('antes'), 'the post below the cut counts') t.absent(texts.includes('depois'), 'the post past the cut does not') }) + +test('the owner confirms a cut writer only once the seeder holds its leave past the cut (#350)', async (t) => { + t.timeout(240000) + const { bootstrap } = await testnet(t) + const assocKp = generateAssociationKeyPair() + const { seeder, url } = await seederOverWs(t, assocKp, bootstrap) + const a = await member(assocKp, 'Ana') + const b = await member(assocKp, 'Bia') + const { community } = await ops.communityCreate(a.ctx, { name: 'Bairro', join: 'open' }) + const { group } = await ops.groupCreate(a.ctx, { name: 'Horta', join: 'open', community }) + await ops.groupPost(a.ctx, { group, text: 'oi' }) + t.ok((await once(a, url, [])).ok, 'creator pushed') + await until(() => indexed(seeder, group, group), { name: 'root indexed' }) + await store.addGroupReplica(b.storage, group) + t.ok((await once(b, url, [])).ok, 'B fetched the group and its community') + await ops.communityJoin(b.ctx, { community }) + await ops.groupJoin(b.ctx, { group }) + await ops.groupPost(b.ctx, { group, text: 'antes' }) + const w = await ownWriter(b, group) + const cut = (await held(b, w)).length + await ops.groupPost(b.ctx, { group, text: 'depois' }) + const length = await offlineLeave(b, group) + const diary = await store.openDiary(b.storage, b.ctx.keyPair) + try { + await store.appendRecord(diary, groupCutRecord({ group, writer: w, cut })) + } finally { + await diary.close() + } + + // The seeder holds W below the cut but has not read past it yet. + const read = seeder._groupPastRange + let resume = null + const stalled = new Promise((resolve) => { resume = resolve }) + seeder._groupPastRange = async function (...args) { + await stalled + return read.apply(this, args) + } + const events = [] + const first = await once(b, url, events, { timeoutMs: 15000 }) + t.absent(first.ok, 'B\'s run does not confirm W at its cut alone') + t.ok(first.errors.join('; ').includes(w + ':push ' + length), 'it waits for W: ' + JSON.stringify(first.errors)) + t.absent(events.some((e) => e.event === 'group-pushed' && e.key === w), 'and never reports it pushed') + t.absent(await seeder._groupServed.get(w).core.has(length - 1), 'control: the seeder lacks the leave') + + resume() + const again = [] + const second = await once(b, url, again) + t.ok(second.ok, 'once the seeder reads past the cut the run ends ok: ' + JSON.stringify(second.errors)) + t.ok(again.some((e) => e.event === 'group-pushed' && e.key === w && e.length === length), 'W reported pushed at its length') + t.ok(await seeder._groupServed.get(w).core.has(length - 1), 'with the leave at the seeder') +}) From cdd9947c4ae8637a420721db48306566432a370a Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 03:07:30 -0300 Subject: [PATCH 16/25] fix(web): stamp the runs of blocks a replica holds past its prefix A replica with more than 4096 blocks between its contiguous length and its signed length got no stamp, so a group with a long released stub or a writer cut early lost its fold memo for good. The stamp now names the runs of held blocks, read from the bitfield, and is null only past 4096 runs. --- packages/core/lib/sparse-held.js | 35 ++++++++++++++++++++- packages/web-member/lib/member-ops.js | 16 ++++++---- packages/web-member/test/group-memo.test.js | 32 +++++++++++++++++++ 3 files changed, 76 insertions(+), 7 deletions(-) diff --git a/packages/core/lib/sparse-held.js b/packages/core/lib/sparse-held.js index cc426739..77bf6ad9 100644 --- a/packages/core/lib/sparse-held.js +++ b/packages/core/lib/sparse-held.js @@ -149,4 +149,37 @@ async function keptPastSeqs (core, from, to) { return out } -module.exports = { advertisedHeld, peerAnswered, nextHeld, peerHolds, askHeld, keptPastSeqs, BITFIELD_SEGMENT } +// The runs of blocks of [from, to) `core` holds, as [start, end) pairs, or +// null past `max` runs. Without the bitfield (heldBits null) it asks has() +// per block, and only over at most `max` blocks (null past that). +async function heldRuns (core, from, to, max) { + const runs = [] + const bits = heldBits(core) + if (bits === null) { + if (to - from > max) return null + let start = -1 + for (let i = from; i < to; i++) { + if (await core.has(i)) { + if (start === -1) start = i + } else if (start !== -1) { + runs.push([start, i]) + start = -1 + } + } + if (start !== -1) runs.push([start, to]) + return runs.length > max ? null : runs + } + let i = from + while (i < to) { + const start = bits.firstSet(i) + if (start === -1 || start >= to) break + if (runs.length === max) return null + let end = bits.firstUnset(start) + if (end === -1 || end > to) end = to + runs.push([start, end]) + i = end + } + return runs +} + +module.exports = { advertisedHeld, peerAnswered, nextHeld, heldRuns, peerHolds, askHeld, keptPastSeqs, BITFIELD_SEGMENT } diff --git a/packages/web-member/lib/member-ops.js b/packages/web-member/lib/member-ops.js index d8ae8370..bce20ad8 100644 --- a/packages/web-member/lib/member-ops.js +++ b/packages/web-member/lib/member-ops.js @@ -102,6 +102,7 @@ const { engagementState, isLiked, isReposted } = require('@varanda/core/lib/enga const { verifySnapshot, blockedSet, revokedIssuerSet } = require('@varanda/core/lib/blocklist') const { blockedKeys } = require('@varanda/core/lib/member-block') const { UsageError } = require('@varanda/core/lib/errors') +const { heldRuns } = require('@varanda/core/lib/sparse-held') const { copyLinkCard } = require('@varanda/core/lib/link-card') const { sealOffer, commitment } = require('@varanda/core/lib/testimonials') const grp = require('@varanda/core/lib/group') @@ -2508,8 +2509,8 @@ async function readGroupCore (ctx, key, rec = null) { const MEMO_MAX = 64 const MEMO_TTL_MS = 30000 -// A replica with more missing blocks than this above its contiguous length -// is not signed: its stamp would cost as much as the read. +// A replica holding more runs of blocks than this past its contiguous +// length is not signed: its stamp would cost as much as the read. const MEMO_MAX_HOLES = 4096 // Tests only: VARANDA_MEMO_CHECK=1 turns the check mode on for every ctx // and gives a ctx without a memo one per storage. @@ -2545,13 +2546,15 @@ function groupStamp (core) { } // readVisible also shows the stored blocks past a hole, so a replica's -// stamp names each of them (null: too many to sign). +// stamp names the runs of them (null: too many to sign). A long stub or a +// writer cut early holds a few runs far past its prefix. async function replicaStamp (core) { const contiguous = core.contiguousLength const length = core.length + const runs = await heldRuns(core, contiguous, length, MEMO_MAX_HOLES) + if (runs === null) return null let out = contiguous + ',' + length + ',' + core.fork - if (length - contiguous > MEMO_MAX_HOLES) return null - for (let i = contiguous; i < length; i++) if (await core.has(i)) out += '.' + i + for (const [start, end] of runs) out += '.' + start + '-' + end return out } @@ -6639,5 +6642,6 @@ module.exports = { isNeedsExclusive, needsExclusive, lentCore, - entriesStamp + entriesStamp, + replicaStamp } diff --git a/packages/web-member/test/group-memo.test.js b/packages/web-member/test/group-memo.test.js index 1945c01f..0f73218e 100644 --- a/packages/web-member/test/group-memo.test.js +++ b/packages/web-member/test/group-memo.test.js @@ -151,6 +151,38 @@ test('the entries stamp follows the seq of a sparse read', (t) => { t.not(entriesStamp([{ seq: 0, record: rec }, { seq: 1, record: null }]), entriesStamp(dense), 'a hole is stamped') }) +// A released stub or a writer cut early holds a few blocks far past its +// prefix (#350): its stamp names them, so the fold memo still holds. +test('a replica holding a few blocks far past its prefix keeps a stamp (#350)', async (t) => { + const Hypercore = require('hypercore') + const RAM = require('random-access-memory') + const source = new Hypercore(RAM) + await source.append(Array.from({ length: 6000 }, (_, i) => b4a.from('b' + i))) + const copy = new Hypercore(RAM, source.key) + await copy.ready() + t.teardown(async () => { + await source.close() + await copy.close() + }) + const a = source.replicate(true, { keepAlive: false }) + const b = copy.replicate(false, { keepAlive: false }) + a.pipe(b).pipe(a) + t.teardown(() => { + a.destroy() + b.destroy() + }) + await copy.download({ start: 0, end: 10 }).done() + await copy.download({ start: 5000, end: 5001 }).done() + t.is(copy.contiguousLength, 10) + t.is(copy.length, 6000) + const first = await ops.replicaStamp(copy) + t.ok(first !== null, 'signed with 5990 blocks missing past its prefix') + t.is(await ops.replicaStamp(copy), first, 'the same copy, the same stamp') + await copy.download({ start: 5500, end: 5501 }).done() + const second = await ops.replicaStamp(copy) + t.ok(second !== null && second !== first, 'a block arriving past the hole changes it') +}) + test('a second group.list decodes no block and answers the same', async (t) => { const { a } = await world(t) const first = await counted(a, () => ops.groupList(a.ctx)) From 18d4c3f906e181b36e8d8bba97d7e1de8e92c569 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 03:12:05 -0300 Subject: [PATCH 17/25] perf(web): back off the sparse goal check while nothing changes A tab checked a sparse writer's goal every 100 ms for as long as it was open, each tick reading every unmet goal. The wait now doubles up to 5 s while the held range stays the same, and a download or a peer coming or going brings it back to 100 ms. The sparse tests also describe the owner's push goal past the cut. --- packages/core/test/sync-groups-sparse.test.js | 3 +- packages/web-member/lib/sync.js | 47 +++++++++++++++++-- .../test/sync-groups-sparse.test.js | 25 ++++++++-- 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/packages/core/test/sync-groups-sparse.test.js b/packages/core/test/sync-groups-sparse.test.js index 45c78b5c..d3b5060a 100644 --- a/packages/core/test/sync-groups-sparse.test.js +++ b/packages/core/test/sync-groups-sparse.test.js @@ -117,7 +117,8 @@ test('a fresh member fed only by a seeder folds a sparse frozen writer and a rel await runUntil(b, bootstrap, sparse) t.alike(await blocksHeld(seeder._groupServed.get(w1).core, 0, leave1.seq + 1), [...Array(cut1).keys(), leave1.seq], 'the seeder keeps W1 below its cut and its leave') t.alike(await blocksHeld(seeder._groupServed.get(w2).core, 0, leave2.seq + 1), [0, leave2.seq], 'and W2 as a stub') - // The owner's --once counts each cut writer pushed at its cut. + // The seeder holds each cut writer up to its cut and its leave: the + // owner's --once counts both pushed. const rb = await syncOnce(bootstrap, b, { timeoutMs: 30000 }) t.ok(rb.result.ok, 'B\'s run ends ok before its deadline: ' + rb.result.errors.join('; ')) diff --git a/packages/web-member/lib/sync.js b/packages/web-member/lib/sync.js index 4c0330a4..6c97e42d 100644 --- a/packages/web-member/lib/sync.js +++ b/packages/web-member/lib/sync.js @@ -411,8 +411,11 @@ const GROUP_PAGE_MS = 30000 // asked again at most this often (the channel allows 2 messages a second). const GROUP_PUSH_PAGE_MS = 1000 // A foreign writer held past its prefix is checked again this often while -// its goal waits on what its peers advertise (core sparse-held.js). +// its goal waits on what its peers advertise (core sparse-held.js), and +// less often, up to SPARSE_CHECK_MAX_MS, while nothing changes: a tab runs +// that goal for as long as it is open. const SPARSE_CHECK_MS = 100 +const SPARSE_CHECK_MAX_MS = 5000 // Writer page requests on one link are spaced this much, all groups and // every re-page together (the seeder refills 2 messages a second). const GROUP_REQUEST_MS = 1000 @@ -581,6 +584,14 @@ function earliestJoin (seq, joins, group, writer) { return out } +// The wait before the next check of a sparse writer's goal: SPARSE_CHECK_MS +// once its held range changed (or first), else twice the last one, up to +// SPARSE_CHECK_MAX_MS. +function sparseCheckDelay (previous, changed) { + if (changed || !(previous > 0)) return SPARSE_CHECK_MS + return Math.min(previous * 2, SPARSE_CHECK_MAX_MS) +} + // Whether an own group core's push is confirmed: the replication peer's // evidence is required (peerHeld), and the page length agrees unless the // seeder does not speak groups (pageLength null). @@ -3629,9 +3640,11 @@ async function syncRun ({ stopGrow: null, stopFork: null, stopQuota: null, + stopSparse: null, retryTimer: null, sparse: null, // { length, contiguous, done }: the last advertisedHeld check sparseTimer: null, + sparseDelay: 0, // the wait before the last check (sparseCheckDelay) reason: null, // why it failed hints: new Set() // 'group:diary:seq' listings whose gate failed on the hint } @@ -4254,9 +4267,10 @@ async function syncRun ({ if (ge.stopGrow) ge.stopGrow() if (ge.stopFork) ge.stopFork() if (ge.stopQuota) ge.stopQuota() + if (ge.stopSparse) ge.stopSparse() if (ge.range) ge.range.destroy() clearLater(ge.retryTimer) - ge.stopGrow = ge.stopFork = ge.stopQuota = ge.range = ge.retryTimer = null + ge.stopGrow = ge.stopFork = ge.stopQuota = ge.stopSparse = ge.range = ge.retryTimer = null } function failGroupEntry (ge, reason, { purge }) { @@ -4349,6 +4363,19 @@ async function syncRun ({ } core.on('append', check) ge.stopQuota = () => core.off('append', check) + // A block past the hole, or a peer coming or going, can meet a sparse + // goal whose check backed off. + const nudge = () => { + if (ge.core === core) sparseNudge(ge) + } + core.on('download', nudge) + core.on('peer-add', nudge) + core.on('peer-remove', nudge) + ge.stopSparse = () => { + core.off('download', nudge) + core.off('peer-add', nudge) + core.off('peer-remove', nudge) + } ge.stopFork = watchFork(core, () => { if (ended || (ge.state !== 'active' && ge.state !== 'regate')) return failGroupEntry(ge, 'fork', { purge: true }) @@ -5273,6 +5300,8 @@ async function syncRun ({ const s = ge.sparse if (s && s.length === ge.core.length && s.contiguous === ge.core.contiguousLength && s.done) return true if (ge.sparseTimer || ended) return false + const changed = !s || s.length !== ge.core.length || s.contiguous !== ge.core.contiguousLength + ge.sparseDelay = sparseCheckDelay(ge.sparseDelay, changed) ge.sparseTimer = later(() => { ge.sparseTimer = null const core = ge.core @@ -5284,10 +5313,20 @@ async function syncRun ({ ge.sparse = { length, contiguous, done } evaluate() }, noop) - }, SPARSE_CHECK_MS) + }, ge.sparseDelay) return false } + // A check that backed off past SPARSE_CHECK_MS runs again shortly. + function sparseNudge (ge) { + if (!(ge.sparseDelay > SPARSE_CHECK_MS)) return + ge.sparseDelay = 0 + if (!ge.sparseTimer) return + clearLater(ge.sparseTimer) + ge.sparseTimer = null + evaluate() + } + // ---- the follow fold (see the header) ---- // Replays the own diary's follow records from the cursor up to its @@ -5862,4 +5901,4 @@ async function syncRun ({ } } -module.exports = { runSync, retryWait, groupVerdict, boxMismatch, addJoins, earliestJoin, groupPushHeld, ownGroupVerdict, bindingRecheck, forgetListed, evictionPurges, capVictim, capExcess, storedJoinProof, refineJoinSeq, gateVerdict, trackPresent, linearExcept, MAX_RANGES, NEWEST_BLOCKS, TAIL_MAX, HEAD_LOW, HEAD_STALL_MS, GATE_TIMEOUT_MS, CONNECT_TIMEOUT_MS, RETRY_FAST_MAX_MS, RETRY_FAST_WINDOW_MS, RETRY_JITTER, MAX_PARALLEL_GATES, MAX_PARALLEL_OPENS, SEEDER_UNAVAILABLE, DIARY_UNAVAILABLE } +module.exports = { runSync, retryWait, sparseCheckDelay, SPARSE_CHECK_MS, SPARSE_CHECK_MAX_MS, groupVerdict, boxMismatch, addJoins, earliestJoin, groupPushHeld, ownGroupVerdict, bindingRecheck, forgetListed, evictionPurges, capVictim, capExcess, storedJoinProof, refineJoinSeq, gateVerdict, trackPresent, linearExcept, MAX_RANGES, NEWEST_BLOCKS, TAIL_MAX, HEAD_LOW, HEAD_STALL_MS, GATE_TIMEOUT_MS, CONNECT_TIMEOUT_MS, RETRY_FAST_MAX_MS, RETRY_FAST_WINDOW_MS, RETRY_JITTER, MAX_PARALLEL_GATES, MAX_PARALLEL_OPENS, SEEDER_UNAVAILABLE, DIARY_UNAVAILABLE } diff --git a/packages/web-member/test/sync-groups-sparse.test.js b/packages/web-member/test/sync-groups-sparse.test.js index 38a4c3f8..da9a17cf 100644 --- a/packages/web-member/test/sync-groups-sparse.test.js +++ b/packages/web-member/test/sync-groups-sparse.test.js @@ -1,7 +1,8 @@ // A seeder keeps a writer its diary cut sparse (#350, docs/protocol.md // "Sparse copies"): below the cut, then only its leaves and key blocks. The -// owner's --once counts that writer pushed at its cut, and a fresh browser -// member fed by that seeder alone ends its --once holding the sparse copy. +// owner's --once counts that writer pushed once the seeder holds it up to +// its cut and its leaves past it, and a fresh browser member fed by that +// seeder alone ends its --once holding the sparse copy. const test = require('brittle') const b4a = require('b4a') const { testnet, until } = require('./helpers/net') @@ -49,6 +50,23 @@ async function offlineLeave (m, group) { } } +// A tab checks a sparse writer's goal for as long as it is open: the checks +// space out while its held range stays the same, and come back to the +// shortest wait once it changes. +test('a sparse goal is checked less often while nothing changes (#350)', (t) => { + const { sparseCheckDelay, SPARSE_CHECK_MS, SPARSE_CHECK_MAX_MS } = require('../lib/sync') + t.is(sparseCheckDelay(0, false), SPARSE_CHECK_MS, 'the first check') + t.is(sparseCheckDelay(0, true), SPARSE_CHECK_MS) + const waits = [] + let d = 0 + for (let i = 0; i < 12; i++) waits.push((d = sparseCheckDelay(d, i === 0))) + t.alike(waits.slice(0, 4), [SPARSE_CHECK_MS, 2 * SPARSE_CHECK_MS, 4 * SPARSE_CHECK_MS, 8 * SPARSE_CHECK_MS], 'twice the last wait while unchanged') + t.is(waits[waits.length - 1], SPARSE_CHECK_MAX_MS, 'up to the bound') + t.ok(waits.every((w) => w <= SPARSE_CHECK_MAX_MS)) + t.ok(SPARSE_CHECK_MAX_MS <= 5000, 'a goal met is noticed within 5 s') + t.is(sparseCheckDelay(SPARSE_CHECK_MAX_MS, true), SPARSE_CHECK_MS, 'a change brings it back') +}) + test('a sparse cut writer: the owner\'s and a fresh reader\'s --once end ok (#350)', async (t) => { t.timeout(240000) const { bootstrap } = await testnet(t) @@ -96,7 +114,8 @@ test('a sparse cut writer: the owner\'s and a fresh reader\'s --once end ok (#35 } t.alike(await blocksHeld(seeder._groupServed.get(w).core, 0, length), [...Array(cut).keys(), length - 1], 'the seeder keeps W below its cut and its leave') - // The owner's --once counts W pushed at its cut. + // The seeder holds W up to its cut and its leave: the owner's --once + // counts it pushed. const rb = await once(b, url, []) t.ok(rb.ok, 'B\'s run ends ok before its deadline: ' + JSON.stringify(rb.errors)) From 9b2137f89598b776c88a72aecc65f5a188599950 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 03:19:57 -0300 Subject: [PATCH 18/25] docs: describe the third round of sparse cut writer fixes Protocol: a chunk past a cut ends at what its synced peers advertise and is read again later, a peer answers by bitfield segment, the owner waits for its own leaves and key blocks past the cut, the memo stamp names runs and the browser's sparse goal check backs off up to 5 s. The changelog and the verification file follow, with the third round's runs and mutation proofs. --- docs/changelog/feat-writer-slot-sparse.md | 2 +- docs/protocol.md | 6 +- docs/verification/feat-writer-slot-sparse.md | 63 ++++++++++++++++++-- 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/docs/changelog/feat-writer-slot-sparse.md b/docs/changelog/feat-writer-slot-sparse.md index 4a76b34d..080090de 100644 --- a/docs/changelog/feat-writer-slot-sparse.md +++ b/docs/changelog/feat-writer-slot-sparse.md @@ -1,3 +1,3 @@ ### Alterado -- Os semeadores passam a guardar só o que conta de um escritor cortado pelo diário (PR `feat/writer-slot-sparse`, parte da #350): os blocos abaixo do corte e, a partir dele, só as saídas do grupo e os blocos de chave. De um escritor liberado (corte 0) fica um toco: o bloco 0, as saídas e os blocos de chave. A cota conta o que fica guardado. O nó core, o nó do navegador e o serviço de contas leem um escritor além do primeiro buraco (só as saídas e os blocos de chave), então quem só fala com semeadores vê o grupo como quem tem as cópias inteiras. Um bloco de chave que cita um escritor de comunidade cortado vale pelo comprimento que o escritor assinou. O escritor sem caixa trocado num grupo privado, depois de liberado, também fica como toco no semeador. O `sync --once` (nó core e navegador) termina com um escritor guardado esparso quando todo bloco que os pares anunciam está guardado, e o dono dá o escritor cortado por enviado no corte; entrar, postar ou selar pela Página num grupo com escritor esparso não responde mais `indisponivel`. Conferência em `docs/verification/feat-writer-slot-sparse.md`. +- Os semeadores passam a guardar só o que conta de um escritor cortado pelo diário (PR `feat/writer-slot-sparse`, parte da #350): os blocos abaixo do corte e, a partir dele, só as saídas do grupo e os blocos de chave. De um escritor liberado (corte 0) fica um toco: o bloco 0, as saídas e os blocos de chave. A cota conta o que fica guardado. O nó core, o nó do navegador e o serviço de contas leem um escritor além do primeiro buraco (só as saídas e os blocos de chave), então quem só fala com semeadores vê o grupo como quem tem as cópias inteiras. Um bloco de chave que cita um escritor de comunidade cortado vale pelo comprimento que o escritor assinou. O escritor sem caixa trocado num grupo privado, depois de liberado, também fica como toco no semeador. O `sync --once` (nó core e navegador) termina com um escritor guardado esparso quando todo bloco que os pares anunciam está guardado, e o dono dá o escritor cortado por enviado quando o semeador o guarda até o corte e já tem as saídas e os blocos de chave dele depois do corte. Um semeador que só recebe o escritor de pares que o guardam esparso também chega às saídas depois do corte. Entrar, postar ou selar pela Página num grupo com escritor esparso não responde mais `indisponivel`. Conferência em `docs/verification/feat-writer-slot-sparse.md`. diff --git a/docs/protocol.md b/docs/protocol.md index 6da6f7ae..0f685abd 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -1921,7 +1921,7 @@ A CW record (admit, ban, hide or invites) whose `group` names a key outside C's Binding drops the records of w at seq >= cut(w), as if w's core ended there, with three exceptions that always stay, whatever the cut: block 0 (the header and a rejoining writer's rejoin node), every `group-leave` (such a cut never undoes a leave, never revives a diary that left and never breaks a rejoin naming that leave, so `removals` and `leaves` are unchanged), and every `group-key` block. A leave written after the mark counts like one written before it (owner decision, 16/11/2026): a leave only takes, never gives, and a leave made on a lost device shortly before it was lost is not undone; the accepted limit is that a stolen device already offline can still take the person out of a group or an admin role, and the person asks to join again. A cut at 0 is the same rule with nothing kept below the cut (owner decision, 16/11/2026): w keeps block 0, every leave and every key block, stays a candidate and is awaited ("Limits", slot walk), and a rejoin naming one of its leaves stays valid. This holds for every mark, not only for honest nodes: whoever can sign d's diary (a thief with the unlocked identity included) can cut the owner's other device at 0 and free its slot, but never undo its leave. The records stop at the lowest cut among every valid mark of d on w the reader holds, even one the slot walk ignored for the slot (written before the join it reads, or held by a reader whose hole hides that join): the walk decides the slot, never the records. A key block past the cut stays valid for reading only: its keys open history as before, but it never sends and adds no recipient to rule 4 of "Eligibility". Everything else past the cut (posts, reactions, topics, polls, choices, events, hides, admits, bans, admin grants, meta, invite flags, sealed envelopes) does not count, and a reference into it is ignored like any reference past a core's end. Every later rule (order, authority, `lastSeq`, keys) sees the shortened log, except that a G key block whose `auth` cites a cut CW is judged by `auth.seq` against the CW's signed length ("Sparse copies", below), so the history it handed out stays readable, from a copy trimmed at the cut too; it still gives no sending authority, since a cut CW is not in C's `authorityWriters`. A cut writer is in neither `liveWriters` nor `authorityWriters`. Its slot follows the slot walk ("Limits"): a cut >= 1 freezes it (still counted, after the live writers of its tier under the 512 cap), a cut 0 releases it, each while there is room; with no room it keeps its slot, cut. The fold returns `cuts` (writer -> cut). The result is the same for any order or arrival of the marks, before or after a rotation. -**Sparse copies** (#350). A seeder keeps a cut writer sparse ("Seeder group admission"): its blocks below the cut, then only its leaves and key blocks, so a reader may hold a writer up to a hole and some blocks past it. Every reader (the core node, the browser node and the account service) reads a writer up to its first missing block and, past it, only the `group-leave` and `group-key` blocks it holds, up to `MAX_LOG_BLOCKS` (`readHeldBlocks` in `store.js` and `node-store.js`), and hands the fold each copy's `signed`: the core length its writer signed, against which Hypercore verified every block held. Roots are still read as a prefix. The fold takes, of a writer, the records below the first seq no copy holds and, past that hole, only its leaves and key blocks, and only when the hole is at or past the writer's cut (`max(cut, 1)`, so a released writer needs its block 0): a sparse copy of a cut writer folds exactly like the whole copy, and a partial copy, cut or not, folds as the prefix it always was. A block past a hole below the cut never counts, since the whole copy would count the records missing before it. A cut writer's signed length is the largest `signed` among its copies; a key block whose `auth` cites a cut CW is valid for reading when `auth.seq` is at most that length (or the CW's held length, when greater), so a CW trimmed at its cut reads the history the whole copy reads. The browser node's fold memo stamps the blocks held past the contiguous length, so a block that arrives past a hole folds again. +**Sparse copies** (#350). A seeder keeps a cut writer sparse ("Seeder group admission"): its blocks below the cut, then only its leaves and key blocks, so a reader may hold a writer up to a hole and some blocks past it. Every reader (the core node, the browser node and the account service) reads a writer up to its first missing block and, past it, only the `group-leave` and `group-key` blocks it holds, up to `MAX_LOG_BLOCKS` (`readHeldBlocks` in `store.js` and `node-store.js`), and hands the fold each copy's `signed`: the core length its writer signed, against which Hypercore verified every block held. Roots are still read as a prefix. The fold takes, of a writer, the records below the first seq no copy holds and, past that hole, only its leaves and key blocks, and only when the hole is at or past the writer's cut (`max(cut, 1)`, so a released writer needs its block 0): a sparse copy of a cut writer folds exactly like the whole copy, and a partial copy, cut or not, folds as the prefix it always was. A block past a hole below the cut never counts, since the whole copy would count the records missing before it. A cut writer's signed length is the largest `signed` among its copies; a key block whose `auth` cites a cut CW is valid for reading when `auth.seq` is at most that length (or the CW's held length, when greater), so a CW trimmed at its cut reads the history the whole copy reads. The browser node's fold memo stamps the runs of blocks held past the contiguous length, read from the bitfield (no stamp, so no memo, past 4096 runs), so a block that arrives past a hole folds again. A `{ gap: true }` entry cuts nothing (the root rule under "Creator marks on group roots" keeps its own gap handling): a hole only delays a mark until it fills, and treating it as a cut would let any hole silence every device of the diary. The fold reports such diaries in `cutUnknown` (only those binding a counted writer), and the slot walk's gap rule ("Limits") sets `incomplete`. A node must not take the fold as complete about cuts for them: it keeps fetching that diary before deciding anything that depends on a writer of it being uncut, for instance before rotating away from it or treating its device as current. @@ -2138,7 +2138,7 @@ Mirrors "Seeder media admission" (section 5) with the same invariants, plus the - **Root G.** Admitted by the same steps as a writer (including quarantine, persistence before the open and the held blocklist) when its block 0 is a valid `group-header` and some served, gated, unblocked diary has a live join for G (the creator's join with `writer == group` is the normal path). **Root binding.** The header's diary counts as the root's creator (and as support) only when that diary is served, not dropped, not blocked and has its own `on: true` `group-join` for writer = G. Otherwise the root is supported only by real member joins, and `/index/group/` answers `creator: null` (also during the startup window before the creator's scan). **Support:** a served root stays while at least one served, unblocked diary keeps it: the creator if it bound the root, a diary with a bound writer in it, or a live join. Withdrawing every join does not drop a root that still has a bound writer. At startup a root does not survive just because its creator's diary is kept: decisions that find a root unsupported are deferred until the startup scans of the served diaries end, and it is judged after them; an unsupported root is then dropped with reason `unbound`. **Blocked or revoked creator:** a served root is purged for a blocked diary, or one whose certificate issuer is revoked, only when that diary BOUND the root (`bound: true`: its header diary has its own `on: true` join for writer = G). Reason `blocked` (also during the startup window), the root stays refused at admission and at startup while its creator stays gone, even while members support the group. A root whose header merely names a blocked diary that never bound it has no creator and stays while members support it. A creator that is both blocked and from a revoked issuer has that issuer recorded too (`revokedCreators`, and `goneRoots` for its bound roots), and a stored issuer is never replaced by null: lifting only the block keeps its bound roots refused, also after a restart, while the issuer stays revoked. A creator dropped or refused for a revoked issuer is remembered with that issuer (`revokedCreators`, a moderation outcome, also across restarts) and judged against the CURRENT revoked-issuer set: adopting a list that lifts the revocation lifts it. An announce of a revoked-issuer certificate is not remembered at all, not even in memory: only a diary's own block-0 verdict (refused at startup, or dropped as revoked-issuer while served) feeds `revokedCreators`, a `goneRoots` issuer and the refusal of a root at admission for a revoked creator, so a root whose creator was never served on this seeder is not refused for a revoked issuer (the root only names that diary). A repeated revoked announce of the same key from one peer counts as a bad announce; the first does not. A purged root is remembered with its creator and, for a revocation, its issuer (`goneRoots`) and refused again while that creator stays gone; lifting the block or revocation lets the same join admit it, and a root admitted again after that keeps `bound`. A diary admitted with a verifying certificate clears its stale revoked entries (`revokedCreators` and the `goneRoots` entries naming it with an issuer). A root whose header names a currently blocked diary, or a diary whose known issuer is revoked, is refused at admission even without a purge record (a root admitted before the block stays until its binding says otherwise). Roots bound before this version get `bound` only after their creator's next scan. A root its creator bound stays listed on `/index/group/` (and in the writer pages) under that creator diary while members keep it served after the creator is dropped for a reason other than moderation (quota, fork, seeder full), also after a restart; `header.diary` stays `null` then, so no creator authority is shown. Members' own writers stay under the rules above. **Issuer kept with the binding.** A bound root's binding keeps the issuer of its creator's block-0 certificate (`issuer`, taken at the creator's scan). When that issuer is revoked under the current list and the creator is not served, the bound root is purged with reason `revoked-issuer` and remembered like any revoked-issuer drop (`revokedCreators`, `goneRoots`), also at startup and also when the creator left first for another reason, such as its quota, so a member that keeps the root served cannot outlast the revocation. **Header fields kept.** While any core of group G is served or being admitted, the seeder keeps the verified header fields of its root (`kind`, `visibility`, `join`, `community`, `member`, and `name` for a public root) in `groupHeaders`, also after the root was purged or refused or its block 0 is withheld, and drops them once no core of G is left or G is blocked; past the bound the record is not kept. A persisted record without `community` and `member` is a protocol v1 leftover and is dropped at start. - **Protocol v2 admission and scan.** A header whose validator returns `legacy` is `header-invalid` (no new wire reason), and the startup re-gate purges every stored v1 root and writer with that reason. A group root must carry a hex64 `community`; the seeder never checks a writer's `member` or any authorization (the fold does). CWs are admitted like any writer and take part in the writer pages. For a served community root the index folds `group-meta`, `group-admin` and `group-create` from the root only (creates are indexed from C's root alone; a CW is not scanned for them, `group-create` being `root-only` there). A public group root is scanned for `group-meta` only (one scan at a time across group roots, skipping blocks without the bytes `group-meta`) and a private root is not scanned. **Denial bounds.** `revokedCreators` and `goneRoots` never evict an entry still in force: when a map reaches 4096 entries the lifted ones go first, and `admitted.json` may hold up to 8192 entries in each. While 4096 or more denials are in force in either map, or after a denial found no room (past 8192), the seeder refuses every group root with the group-rejected reason `denials-full` and logs `{ event: 'warning', reason: 'denials-full' }`, so a purged root cannot return for lack of memory. -- **Cut writers** (#350, owner decision). A writer its diary cut (frozen, released, or live with a cut once the frozen slots or the releases are spent: `groupWalkCut` over the slot walk) keeps its blocks below the cut, block 0 always, and from the cut on only its `group-leave` and `group-key` blocks, the ones the fold still counts there and readers read past a hole ("Diary cuts on own writers", "Sparse copies"). A released writer (cut 0) is a stub: block 0, its leaves and its key blocks. Its download stops at the cut; the blocks past it, up to `MAX_LOG_BLOCKS`, are fetched `CUT_CHUNK` = 256 at a time, read for their type only (`decodeBlock`, never validated) and cleared unless kept. Hypercore keeps the tree nodes the proofs of the kept blocks need, so a reader verifies each against the signed length. The binding in admitted.json carries `cut`, `clearedFrom` and `clearedTo` (the range past the cut already read) and is persisted before a chunk is cleared; the next serve reads that range again and clears what it still holds beyond the kept blocks, so a crash in between leaves nothing extra for good. The quota counts what is kept: at admission and at startup, the blocks below the cut (`min(max(cut, 1), length)` against `maxLogBlocks`); while the whole log is over the quota, the blocks below the cut and those kept past it, and their bytes, against `maxLogBlocks` and `maxLogBytes`, counted as they are read. A writer past it is dropped with `quota` and its join is refused as at admission. The caps above count a stub like any writer. The diary's scan gives the cut: a lower one applies at once (also to a core still opening), a higher one or none only once the scan caught up with the diary (a cut restored from admitted.json whose record was deleted, for example), and then the download reaches the new cut, or the whole log, so what clearing removed is fetched again. A cut restored at startup clears nothing new until the scan confirms it. A join refused before (`quota`, `header-invalid`) is tried again once per lower cut. A writer of a private group whose header has no box (the one a node replaced, "Writers named by a node") is `header-invalid` while counted and kept as a stub once released, so its block 0 stays available to readers, which reject it by its header ("Limits"); if its release goes away it is judged again and dropped. +- **Cut writers** (#350, owner decision). A writer its diary cut (frozen, released, or live with a cut once the frozen slots or the releases are spent: `groupWalkCut` over the slot walk) keeps its blocks below the cut, block 0 always, and from the cut on only its `group-leave` and `group-key` blocks, the ones the fold still counts there and readers read past a hole ("Diary cuts on own writers", "Sparse copies"). A released writer (cut 0) is a stub: block 0, its leaves and its key blocks. Its download stops at the cut; the blocks past it, up to `MAX_LOG_BLOCKS`, are fetched `CUT_CHUNK` = 256 at a time, read for their type only (`decodeBlock`, never validated) and cleared unless kept. Hypercore keeps the tree nodes the proofs of the kept blocks need, so a reader verifies each against the signed length. The binding in admitted.json carries `cut`, `clearedFrom` and `clearedTo` (the range past the cut already read) and is persisted before a chunk is cleared; the next serve reads that range again and clears what it still holds beyond the kept blocks, so a crash in between leaves nothing extra for good. A chunk ends once every block its synced peers advertise there is held (`advertisedHeld`, "Completion goals"), even when they lack some of it: a seeder fed only by peers that keep the writer sparse (another seeder, or after a `quota` drop that a lower cut lifted) still reads past the first chunk. What it holds there is kept or cleared as above, and a chunk read that way joins the binding's unsure range (`unsureFrom`, `unsureTo`, within the range already read), read again chunk by chunk 1 s after a peer joins the core and at intervals doubling from 1 s to 5 minutes; a chunk read again whole leaves it, so a leave or key block those peers lacked comes from a fuller peer. The quota counts what is kept: at admission and at startup, the blocks below the cut (`min(max(cut, 1), length)` against `maxLogBlocks`); while the whole log is over the quota, the blocks below the cut and those kept past it, and their bytes, against `maxLogBlocks` and `maxLogBytes`, counted as they are read. A writer past it is dropped with `quota` and its join is refused as at admission. The caps above count a stub like any writer. The diary's scan gives the cut: a lower one applies at once (also to a core still opening), a higher one or none only once the scan caught up with the diary (a cut restored from admitted.json whose record was deleted, for example), and then the download reaches the new cut, or the whole log, so what clearing removed is fetched again. A cut restored at startup clears nothing new until the scan confirms it. A join refused before (`quota`, `header-invalid`) is tried again once per lower cut. A writer of a private group whose header has no box (the one a node replaced, "Writers named by a node") is `header-invalid` while counted and kept as a stub once released, so its block 0 stays available to readers, which reject it by its header ("Limits"); if its release goes away it is judged again and dropped. - **Drops** (purge before un-listing, runtime order of media (e), each logs `{ event: 'group-rejected', key, diary, group, reason }`): the diary is dropped or blocked (D's writers go with it, with `blocked` or the diary's own reason; the root stays while supported), the root key is in the blocklist snapshot (`group-blocked`, for the root and every writer of that group; the blocklist apply loop also walks the group map, because it otherwise walks only `_served`), a fork (`fork`), the quota (`quota`), an unsupported root (`unbound`), a root whose creator (the header diary, when it bound the root) is blocked or has a revoked issuer (`blocked`) or a failed header re-check (`header-invalid`). NEVER on a `group-leave`, never on a join turned off and never on a cut: a cut or released writer stays held whole ("Cut writers"). - **Retention.** Index withdrawal and membership end are distinct, as defined above. Admitted writer and root cores stay served and listed across either event (the index only marks the writer `left`), preserving the moderation records of departed members. - **Startup** re-gates listed group cores like listed media cores (a core whose binding is gone, or whose root is blocked, is purged first), applying the same per-diary counts to persisted bindings (32 per group); for bindings written before the cap that count follows iteration order, not earliest join. A cut writer, a released one included, is restored whole (quota on the whole log), and the scans recompute cuts and releases from the diaries. `seed list` prints the group cores indented under their diary (`grupo (grupo ) blocos bytes`), `seed drop` purges them as auxiliaries, and `stats` adds `groupCores`. @@ -2180,7 +2180,7 @@ Added to section 7 "Seeder index route", under the same caps, Host check, 405 an - **Round 13, core node.** (1) A new writer key of a group is refused (`group-skipped` reason `quota`) while the group's foreign writer cores in the admission table plus those stored on disk, including writers kept after a quota stop, reach `MAX_GROUP_WRITERS` (`runSync` option `maxGroupWriters`; the group root is no writer slot and is never refused by this cap); a stored writer keeps its place, and a refused key is gated again once a place frees. (2) Block-0 divergences of group writers become one hard-error line `bloco 0 divergente em escritores de grupo: ()` (the result is still not ok), and a run keeps at most 32 hard-error strings. - **Membership and purges.** A fetched historical join block proves binding only. Current membership comes from the writer fold above. A stored writer core is purged only for an ACCEPTED writer (one that passed the gate): when D is blocked or its issuer is revoked, on a fork, or when its local block 0 differs from the gated one. A bad hint from a seeder never purges an unaccepted core. A blocked group root fails and purges every foreign core of that group with reason `blocked`, in the core node and in the browser alike; the member's own cores are never touched, and what is stored can still be read locally until it is purged. - **Binding diary release (core node).** A group writer's binding diary is downloaded and kept in the main store only while something needs it: a follow holds it, a writer in the admission table names it, or a stored foreign writer or root names it (the census is rebuilt from disk at start). When a writer is purged, retires or is evicted and nothing else needs its diary, the node stops downloading the diary and purges it from the main store. A writer stopped for quota keeps its diary (and its blocks). At run start a binding diary recorded in `group-certs.json` that no stored core names and that no follow holds is purged. At core node start, a member of any group also purges an unfollowed, unlisted diary unless a stored group writer or root names it or `group-certs.json` records it; that cleanup always rebuilds the census first, also when `groups.json` and `group-certs.json` are empty, and never purges a group core this node can write. A `group-join` block that its author deleted (section 2 "Clearing deleted bytes") withdraws the writer it bound: the core node drops the writer from the run (`group-skipped` reason `withdrawn`) without purging its blocks, and the group goal stops waiting for it. At browser node start, a stored replica that a stored group core binds (or that the run holds as a binding diary) stays indexed and is never purged by the unfollowed-replica cleanup. -- **Completion goals** (`--once` and the browser's once runs). Each group goal runs once per run and is met when (a) every writer page of every joined group was fetched (not `incomplete`) or the channel timed out, (b) every admitted foreign writer, and the diary that binds it up to its join, reached the length its seeder advertised, exactly like followed replicas (on the core node a core admitted through quarantine counts only after the block 0 its main store received was compared with the gated one), and (c) own writers were pushed to the length the seeder advertises for them (browser: confirmed by `group-pushed { key, length }`); all bounded by the run's deadline. A writer a seeder keeps sparse ("Sparse copies") never reaches its signed length contiguously, so (b) also holds for a foreign writer held up to the advertised length and past it with a hole once every connected peer that synced has answered for the blocks below the signed length (up to `MAX_LOG_BLOCKS`) and every block one of them advertises there is held (`advertisedHeld` in `sparse-held.js`, checked again every `SPARSE_CHECK_MS` = 100 ms while the goal waits). Past a peer's contiguous length its bitfield arrives only as the answer to a download range's want, so a peer counts as having answered once a page of its bitfield covering that range arrived. (c) for an own writer the own diary cut asks the seeder to hold it up to `max(cut, 1)` (`diaryWriterCut` over the diary's slot walk: 0 once released), never past the length the device holds; the core node reads the cuts when the run opens its writers, the browser at each scan of its diary. The account service's group fold for a page (`apps/cli/lib/account/pages-groups.js`, section 12) counts a listed writer whole on the same rule, so a group with such a writer is no longer partial (`indisponivel`): its fetch ends, before `FETCH_TIMEOUT_MS`, once every block the seeder advertises below the signed length is held (polled every 50 ms). Roots and diaries keep the contiguous rule. Against a seeder without `varanda/groups/1` (groups unsupported) the browser's own group pushes are not once-run goals, so a once run finishes. The goal of a binding diary is met at its present length (section 2 "Clearing deleted bytes"), so a deleted post no longer keeps a once run open, and join blocks after a cleared hole are read. Group writer and root cores keep their own tracking, since no delete records exist for them. Goals G1 to G3 of section 6 are unchanged. The browser's write gate for a group write confirms the own writer's length, not only the diary. For each community the goals are C's root, every page of its CWs, and the binding diaries, as for groups. +- **Completion goals** (`--once` and the browser's once runs). Each group goal runs once per run and is met when (a) every writer page of every joined group was fetched (not `incomplete`) or the channel timed out, (b) every admitted foreign writer, and the diary that binds it up to its join, reached the length its seeder advertised, exactly like followed replicas (on the core node a core admitted through quarantine counts only after the block 0 its main store received was compared with the gated one), and (c) own writers were pushed to the length the seeder advertises for them (browser: confirmed by `group-pushed { key, length }`); all bounded by the run's deadline. A writer a seeder keeps sparse ("Sparse copies") never reaches its signed length contiguously, so (b) also holds for a foreign writer held up to the advertised length and past it with a hole once every connected peer that synced has answered for the blocks below the signed length (up to `MAX_LOG_BLOCKS`) and every block one of them advertises there is held (`advertisedHeld` in `sparse-held.js`, checked again every `SPARSE_CHECK_MS` = 100 ms while the goal waits; in the browser, which may run that goal for as long as a tab is open, the wait doubles up to 5 s while the copy's held range stays the same, and a download or a peer coming or going brings it back to 100 ms). Past a peer's contiguous length its bitfield arrives only as the answer to a download range's want: one want per bitfield segment (2097152 blocks), answered from the segment's first block up to the last page the peer holds anything in. So a peer counts as having answered once the first page of each segment of that range arrived; a later page it holds nothing in never comes. (c) for an own writer the own diary cut asks the seeder to hold it up to `max(cut, 1)` (`diaryWriterCut` over the diary's slot walk: 0 once released), never past the length the device holds, and past the cut each of its own `group-leave` and `group-key` blocks, as the seeder's replication peer advertises it or as served to that peer on this connection (a peer never announces a block it gets to one that already holds it). The device asks that peer for its bitfield there (`askHeld` in `sparse-held.js`), since a full copy sends no wants of its own; the core node reads the cuts when the run opens its writers, the browser at each scan of its diary. The account service's group fold for a page (`apps/cli/lib/account/pages-groups.js`, section 12) counts a listed writer whole on the same rule, so a group with such a writer is no longer partial (`indisponivel`): its fetch ends, before `FETCH_TIMEOUT_MS`, once every block the seeder advertises below the signed length is held (polled every 50 ms). Roots and diaries keep the contiguous rule. Against a seeder without `varanda/groups/1` (groups unsupported) the browser's own group pushes are not once-run goals, so a once run finishes. The goal of a binding diary is met at its present length (section 2 "Clearing deleted bytes"), so a deleted post no longer keeps a once run open, and join blocks after a cleared hole are read. Group writer and root cores keep their own tracking, since no delete records exist for them. Goals G1 to G3 of section 6 are unchanged. The browser's write gate for a group write confirms the own writer's length, not only the diary. For each community the goals are C's root, every page of its CWs, and the binding diaries, as for groups. - **Parent paging.** For every local group root held the node reads its header's `community`, and for every local community its key. For each community C it pages C on the groups channel (the root and its CWs), gates each CW like a group writer (a valid header with `group` = C and `member: null`, then the binding diary's block 0 and join block), stores the CWs and their binding diaries, and pages only the roots of C's catalogue (the `/index/community/` rows are a hint, the fold verifies) so that `community.info` can name them. Each root is judged by its own block 0 under a role fixed for the run: a key joined as a group must be a `group` root, a key reached as a parent must be a `community` root, and a `v: 1` header is `header-invalid`; the `group-skipped` reason `not-community` (a parent that is not a community root; the node's own verdict, the seeder is not charged) is kept. A run holds at most groups x 512 plus communities x 512 cores. A node folding a group passes `buildGroup` the parent's input (C's root, its CWs and their joins). The core sync emits `community-parent { key, group }` (`key` the community root C, `group` the group G) the first time a held group root names a community this node does not already hold, and `group-parent { key: C, group: G }` once per local group whose held root names C, held or not (the session uses it to fan `groups.changed` out from C's growth to its groups; the CLI prints neither). - **Diary-bound roots (browser).** Besides kv `groupWriters` and kv `groupReplicas`, the browser sync pages every root its own diary binds now as a replica: the newest `group-join` per writer still `on: true`, among the writers the slot walk counts per group ("Limits"), the first 64 groups by live join seq (`diaryGroupKeys`, the seeder's live-join rule). It reads them at run start from the stored diary and again whenever the own diary is caught up with the seeder, so a second device of the same passkey (same diary, fresh storage) rebuilds its communities and groups from the diary. A writer the diary newly binds to a root already adopted is paged at once (that group alone, ahead of the pass in progress or in a pass of its own), and again after waits of 2, 5 and 10 s (each counted from the previous page, so at about 2, 7 and 17 s) while the seeder does not list it yet; the writers of the run's first scan are not chased, the regular pass asks for them. A diary block the read could not get (not downloaded yet, or a read that failed) is kept, read again on every later scan, when it downloads and, after a failed read, on a short timer, and never holds back the joins after it. A once run does not end while that read of the own length is pending: the roots it adopts are paged before the run's goals are met. Such a root's writers are gated like any replica's, a catalogue child the diary joined is not paged root only (one already paged root only in the run is paged again in full), and a community among them is paged in full with its catalogue. Its own header decides its kind: another root's header naming it as its community never makes it a parent key, so it is never refused as `not-community`. A root no join binds now (every device left) is not adopted, a blocked key never is, and a key with no root at the seeder is dropped by the probe rule and not asked for again in that run. Nothing is appended, no writer is created and nothing is listed in kv for an adopted key until its block 0 is stored (the gate then lists it in kv `groupReplicas`). The core node is unchanged: its diary is per device (section 1), so another device never shares it. - **Automatic device enrolment (browser).** After a sync, a device of the same passkey enrols by itself in each diary-bound root it holds and has no kv `groupWriters` entry for, while the fold says the diary is a `member` there and the diary binds fewer than `MAX_GROUP_WRITERS_PER_DIARY` - 2 (6) writers in it: the last two slots, which the seeder never releases, are spent only by an explicit join. It writes what `community.join` or `group.join` writes (a new writer naming the diary's latest leave, and the diary's public `group-join`), communities first, then a group only when this device's community writer is admitted in C (an open community or its creator; in an approval community the diary's admission covers the new community writer, "Admits list their writers", and its groups follow on a later run, once the fold shows it admitted). Geral is planned like any group of C with its own checks: a diary that left Geral is not put back, and Geral's last two slots stay reserved too. A kv writer the diary never bound (an interrupted join) is bound again, reusing that writer only when this device derives its key from the entry's nonce; otherwise the root is skipped (`foreign`), and an explicit join writes a new writer instead. Only a writer this device derived is ever reused by a join. A root that is left, banned, requested, blocked, unheld, an orphan, not joined by the own diary (an opened link) or not foldable yet (no pending join) is skipped, and no admit or grant is written automatically (the fold's coverage of a new device by the diary's admission writes nothing). Rules in [rpc.md](rpc.md) 5.63. The core node is unchanged (its diary is per device, section 1). diff --git a/docs/verification/feat-writer-slot-sparse.md b/docs/verification/feat-writer-slot-sparse.md index 720d2e93..6f6280ac 100644 --- a/docs/verification/feat-writer-slot-sparse.md +++ b/docs/verification/feat-writer-slot-sparse.md @@ -12,7 +12,7 @@ Dobra (`packages/core/lib/group.js`): Leitores: - Nó core: `readHeldBlocks` em `packages/core/lib/store.js` (o prefixo contíguo e, depois do primeiro buraco, só as saídas e os blocos de chave guardados, até `MAX_LOG_BLOCKS`); `readCoreParts` em `member.js` lê os escritores assim e passa `signed: core.length`. As raízes continuam lidas como prefixo. -- Nó do navegador: `readHeldBlocks` em `packages/web-member/lib/node-store.js`; em `member-ops.js`, `writerCoreState` para os escritores, o carimbo da memória da dobra (`groupStamp = replicaStamp`) conta os blocos guardados depois do comprimento contíguo, `signed` em `foldGroup` e `foldCommunity`, e `foldCommunity` passa a receber as marcas de corte dos diários. +- Nó do navegador: `readHeldBlocks` em `packages/web-member/lib/node-store.js`; em `member-ops.js`, `writerCoreState` para os escritores, o carimbo da memória da dobra (`groupStamp = replicaStamp`) nomeia as faixas de blocos guardados depois do comprimento contíguo (terceira rodada; antes, cada bloco), `signed` em `foldGroup` e `foldCommunity`, e `foldCommunity` passa a receber as marcas de corte dos diários. - Serviço de contas: `apps/cli/lib/account/pages-groups.js` lê os escritores listados com `readHeldBlocks` e passa `signed`. Semeador (`packages/core/lib/seeder.js`): @@ -27,13 +27,23 @@ Semeador (`packages/core/lib/seeder.js`): Metas de conclusão com cópia esparsa (segunda rodada, pontos (a) e (b)): -- `packages/core/lib/sparse-held.js` (novo, roda no Node, no Bare e no navegador): `advertisedHeld(core, end)` é verdadeiro quando algum par conectado já sincronizou, todo par sincronizado já respondeu pelos blocos abaixo de `end` e todo bloco abaixo de `end` que um deles anuncia está guardado. Depois do comprimento contíguo de um par, o bitfield dele só chega como resposta ao pedido de um intervalo de download, então o par conta como respondido quando chegou uma página do bitfield que cobre o intervalo. +- `packages/core/lib/sparse-held.js` (novo, roda no Node, no Bare e no navegador): `advertisedHeld(core, end)` é verdadeiro quando algum par conectado já sincronizou, todo par sincronizado já respondeu pelos blocos abaixo de `end` e todo bloco abaixo de `end` que um deles anuncia está guardado. Depois do comprimento contíguo de um par, o bitfield dele só chega como resposta ao pedido de um intervalo de download, então o par conta como respondido quando chegou a primeira página de cada segmento do bitfield que o intervalo toca (terceira rodada; antes, toda página até o fim). - `diaryWriterCut(entries, G, W)` em `group.js`: o corte que a caminhada do diário dá ao escritor (0 quando liberado, `null` sem corte), igual ao `groupWalkCut` do semeador. -- Nó core (`sync.js`): na meta de `--once`, um escritor alheio guardado até o anunciado e com buraco depois conta como completo quando `advertisedHeld` vale até `min(comprimento assinado, MAX_LOG_BLOCKS)` (`sparseMet`, conferido de novo a cada `SPARSE_CHECK_MS` = 100 ms enquanto a meta espera). O próprio escritor cortado pelo diário conta como enviado quando o semeador o guarda até `max(corte, 1)`, nunca além do comprimento local; os cortes são lidos do diário quando a sincronização abre os escritores. +- Nó core (`sync.js`): na meta de `--once`, um escritor alheio guardado até o anunciado e com buraco depois conta como completo quando `advertisedHeld` vale até `min(comprimento assinado, MAX_LOG_BLOCKS)` (`sparseMet`, conferido de novo a cada `SPARSE_CHECK_MS` = 100 ms enquanto a meta espera). O próprio escritor cortado pelo diário conta como enviado quando o semeador o guarda até `max(corte, 1)`, nunca além do comprimento local, e (terceira rodada) já tem as saídas e os blocos de chave dele depois do corte; os cortes são lidos do diário quando a sincronização abre os escritores. - Nó do navegador (`web-member/lib/sync.js`): as mesmas duas regras (`sparseMet` em `unmetGroupGoals`; `pushTarget` em `checkMedia`), com os cortes relidos a cada varredura do diário (`fold.cutEntries`, `ownCutEntries`), e `checkMedia` roda de novo depois de cada varredura. - Serviço de contas (`pages-groups.js`): o `fetch` de um escritor listado termina, antes de `FETCH_TIMEOUT_MS`, quando todo bloco que o semeador anuncia abaixo do comprimento assinado está guardado (conferido a cada 50 ms), e então a cópia conta como completa. Raízes e diários continuam com a regra do contíguo. -Documentos: `docs/protocol.md` (bit `CAP_WRITER_CUTS`, entrada de `buildGroup` e `buildCommunity` com `signed`, "Slot walk" e "Writers named by a node" sobre o sem caixa, regra do `auth.seq` em "Diary cuts on own writers" e em "Community distributors", parágrafo novo "Sparse copies", "Cut writers" do semeador reescrito; na segunda rodada, "Completion goals" com a cópia esparsa, o próprio escritor cortado e o `fetch` do serviço de contas). `docs/rpc.md` não muda: nenhum método nem resultado mudou. +Documentos: `docs/protocol.md` (bit `CAP_WRITER_CUTS`, entrada de `buildGroup` e `buildCommunity` com `signed`, "Slot walk" e "Writers named by a node" sobre o sem caixa, regra do `auth.seq` em "Diary cuts on own writers" e em "Community distributors", parágrafo novo "Sparse copies", "Cut writers" do semeador reescrito; na segunda rodada, "Completion goals" com a cópia esparsa, o próprio escritor cortado e o `fetch` do serviço de contas). Na terceira rodada, "Cut writers" com o fim do pedaço pelo anunciado e a faixa incerta, "Completion goals" com a regra do segmento, as saídas próprias depois do corte e a espera do navegador, e "Sparse copies" com o carimbo por faixas. `docs/rpc.md` não muda: nenhum método nem resultado mudou. + +Terceira rodada (revisão em `d487e5e7`, sem P1): + +- Semeador, pedaço depois do corte lido de pares esparsos (P2-1, `_groupPastRange` e `_cutGroupCore` em `seeder.js`): um pedaço termina quando todo bloco que os pares sincronizados anunciam ali está guardado (`advertisedHeld`), não só quando os 256 blocos chegaram. O que ele guarda ali é mantido (e contado na cota) ou limpo como antes, e o pedaço entra na faixa incerta da ligação (`unsureFrom`, `unsureTo`, dentro da faixa já lida), relida pedaço a pedaço 1 s depois de um par entrar no núcleo e em intervalos que dobram de 1 s a 5 minutos; um pedaço relido inteiro sai dela. Antes, um semeador que só recebia o escritor de pares esparsos (outro semeador, ou depois de uma saída por `quota` que um corte menor levantou) nunca passava do primeiro pedaço, e uma saída em corte + 256 ou depois nunca era buscada. +- Dono de um escritor cortado (P2-2): o nó core (`ownKeptMissing` em `sync.js`) e o navegador (`seederHoldsKept` em `checkMedia`) só dão o escritor por enviado quando, além de o semeador guardá-lo até `max(corte, 1)`, o par de replicação do semeador anuncia cada `group-leave` e `group-key` próprio do corte em diante, ou recebeu esse bloco nesta conexão (evento `upload`: um par nunca anuncia um bloco que recebe a quem já o tem). `askHeld` (`sparse-held.js`) pede a esse par o bitfield ali, porque uma cópia inteira não manda pedidos próprios. Enquanto falta, o nó core responde `:corte ` e o navegador `:push `, sem `group-pushed`. +- `peerAnswered` (P3-3): um pedido pede um segmento inteiro do bitfield (2097152 blocos), e a resposta traz a primeira página do segmento e as páginas seguintes que o par tem. O par conta como respondido quando chegou a primeira página de cada segmento do intervalo; antes era preciso toda página de 32768 bits até o fim, e um par esparso num escritor longo nunca respondia. +- `readHeldBlocks` (P3-6, `store.js` e `node-store.js`): depois do buraco, pula para o próximo bloco guardado pelo bitfield que o próprio `has()` lê (`firstSet`, presente no Hypercore 10 e no 11, conferido em `node_modules`), em vez de um `has()` por seq. Sessão que não seja a padrão (snapshot, batch) continua com o `has()` por bloco. +- Carimbo da memória da dobra (P3-5, `replicaStamp` em `member-ops.js`): nomeia as faixas de blocos guardados depois do prefixo (`heldRuns` em `sparse-held.js`), nulo só com mais de 4096 faixas. Antes ficava nulo com mais de 4096 blocos entre o contíguo e o comprimento, e um grupo com um toco longo ou um escritor cortado cedo perdia a memória de vez. +- `sparseMet` do navegador (P3-4): a espera entre conferências dobra até 5 s (`SPARSE_CHECK_MAX_MS`) enquanto a faixa guardada da cópia não muda; um download ou um par que entra ou sai volta a 100 ms. Antes eram 100 ms enquanto a aba ficasse aberta. +- Comentários (P3-7, `seeder.js`): `released = (cut === 0)` cobre também o escritor ainda contado cortado em 0 depois de gastas as 16 liberações; os dois ficam como toco. ## Como verificar @@ -78,6 +88,28 @@ O teste de ponta a ponta: A cria a comunidade e dois grupos; B escreve nos dois, Na segunda rodada ele confere também que o `sync --once` do dono (B) e o do leitor novo (R) terminam `ok` antes do prazo de 30 s. `packages/web-member/test/sync-groups-sparse.test.js` (novo, DHT local e semeador atrás de WebSocket) faz o mesmo no nó do navegador com um escritor congelado: o `--once` de B e o de R terminam `ok`, R guarda o escritor abaixo do corte mais a saída, a saída conta e o post depois do corte não. A saída ali é escrita direto no escritor, como faria um aparelho offline: `group.leave` também desliga a entrada no diário, e esse caso esbarra num problema anterior a este PR (ponto (g) abaixo). `apps/cli/test/pages-groups-sparse.test.js` (novo, arquivo próprio porque `pages-groups.test.js` já usa boa parte do tempo por arquivo do executor): com o semeador guardando esparso o escritor congelado de B, a Página entra na comunidade, entra no grupo e posta, e a leitura dela vem com `incomplete: false`, o post abaixo do corte e sem o de depois. Selar (`group-seal`, grupo privado) passa pela mesma conferência `loaded.partial` e não tem teste próprio aqui. +Terceira rodada, cada arquivo sozinho: + +| Arquivo | Testes | Asserções | +|---|---|---| +| `packages/core/test/sparse-held.test.js` (novo) | 2/2 | 8/8 | +| `packages/core/test/sync-groups-sparse.test.js` (teste 2 novo) | 2/2 | 20/20 | +| `packages/core/test/seeder-groups-cuts.test.js` (teste 15 novo) | 15/15 | 336/336 | +| `packages/core/test/sync-groups-3.test.js` | 16/16 | 71/71 | +| `packages/core/test/group-member.test.js` | 13/13 | 151/151 | +| `packages/core/test/group-member-6.test.js` | 10/10 | 72/72 | +| `packages/core/test/sync-private.test.js` | 4/4 | 16/16 | +| `packages/web-member/test/sync-groups-sparse.test.js` (testes 1 e 3 novos) | 3/3 | 27/27 | +| `packages/web-member/test/sync-groups-2.test.js` | 26/26 | 130/130 | +| `packages/web-member/test/private-ops.test.js` | 24/24 | 231/231 | +| `packages/web-member/test/group-memo.test.js` (teste 2 novo) | 12/12 | 65/65 | +| `apps/cli/test/pages-groups-sparse.test.js` | 1/1 | 8/8 | +| `apps/cli/test/pages-groups.test.js` | 3/3 | 153/153 (a asserção sobre a Geral voltou a rodar, `joined.group`, como na primeira rodada) | + +`npm run lint` passa. + +O teste 15 de `seeder-groups-cuts.test.js` monta o caso P2-1: o primeiro semeador guarda W esparso (corte 3; cinco posts, mais 300, a saída e mais um post, então a saída fica depois de corte + 256); B sai, e um segundo semeador recebe W só do primeiro. Ele guarda a saída, conta-a na cota (`keptPast`) e lista como incerta a faixa lida só de pares esparsos (`unsureFrom` 3, `unsureTo` o comprimento); quando B volta com W inteiro, relê a faixa, limpa de novo o que não fica e tira a faixa incerta da ligação. O teste 2 de `packages/core/test/sync-groups-sparse.test.js` e o teste 3 de `packages/web-member/test/sync-groups-sparse.test.js` seguram `_groupPastRange` do semeador: com a saída só no aparelho, o `--once` do dono não termina `ok` e diz o que falta (nó core `:corte `, navegador `:push ` e nenhum `group-pushed`); solto o semeador, termina `ok`. O caso do `peerAnswered` (`packages/core/test/sparse-held.test.js`, teste 1) usa um escritor de 40000 blocos e um par com `[0, 10)` mais o bloco 500, como o repro da revisão. O pulo pelo bitfield do `readHeldBlocks` tem teste só no nó core (teste 2 do mesmo arquivo); o do navegador são as mesmas quatro linhas, sem teste próprio. O teste da espera do `sparseMet` (P3-4) é de unidade, sobre `sparseCheckDelay`; o empurrão por download ou par não tem teste que falhe sem ele, e os testes esparsos do navegador passam com ele. + Não rodados aqui, por regra do brief: a suíte inteira e a demo (a sessão que abriu esta roda a CI de entrega). ## Prova de regressão @@ -125,6 +157,25 @@ Segunda rodada, uma mutação por vez (`node ` com a mudança tirada, d | B1 o `fetch` do escritor listado como antes | `apps/cli/test/pages-groups-sparse.test.js` | tempo esgotado "group join" (60 s de `indisponivel`) | | B2 o `fetch` esparso sem `advertisedHeld` | o mesmo | tempo esgotado "group join" | +Terceira rodada, uma mutação por vez: + +| Mutação | Arquivo | Resultado | +|---|---|---| +| C1 `peerAnswered` exige toda página de 32768 bits até o fim | `packages/core/test/sparse-held.test.js` | 1/2, 7/8, falha "every advertised block is held well before the deadline" | +| F1 `readHeldBlocks` com um `has()` por seq | o mesmo | 1/2, 7/8, falha "has() asked 40000 times, not once per block" | +| P1 pedaço depois do corte termina só com `range.done()` | `packages/core/test/seeder-groups-cuts.test.js` | tempo esgotado "W lido do primeiro semeador" (teste 15) | +| P2 sem a releitura da faixa incerta | o mesmo | tempo esgotado "W relido inteiro" (teste 15) | +| Q1 dono sem conferir as saídas depois do corte (nó core) | `packages/core/test/sync-groups-sparse.test.js` | 1/2, 18/20, falham "B's run does not count W pushed at its cut alone" e "it waits for the leave" | +| Q2 `askHeld` não pede nada (nó core) | o mesmo | 1/2, 19/20, falha no teste 1 "B's run ends ok before its deadline: tempo esgotado: :corte 4, :corte 3" | +| R1 dono sem conferir as saídas (navegador) | `packages/web-member/test/sync-groups-sparse.test.js` | 1/2, 17/20, falham "B's run does not confirm W at its cut alone", "it waits for W: []" e "and never reports it pushed" | +| R2 `askHeld` não pede nada (navegador) | o mesmo | passa, 2/2, 20/20 (abaixo) | +| R3 blocos servidos não contam (navegador) | o mesmo | 1/2, 18/20, falham "once the seeder reads past the cut the run ends ok: tempo esgotado: :push 4" e "W reported pushed at its length" | +| R4 blocos servidos não contam (nó core) | `packages/core/test/sync-groups-sparse.test.js` | passa, 2/2, 20/20 (abaixo) | +| S1 carimbo com o limite antigo de 4096 blocos | `packages/web-member/test/group-memo.test.js` | 11/12, 63/65, falham "signed with 5990 blocks missing past its prefix" e "a block arriving past the hole changes it" | +| S2 espera sempre de 100 ms | `packages/web-member/test/sync-groups-sparse.test.js` | 2/3, 25/27, falham "twice the last wait while unchanged" e "up to the bound" | + +As linhas R e Q rodaram antes do teste de unidade da S2, por isso 2 testes no arquivo do navegador. R2 passa porque, neste teste, o pedido que `seederHoldsMedia` já faz ao conectar (o segmento 0, antes de chegar a faixa contígua) traz o bitfield depois do corte; `askHeld` cobre o caso em que isso não acontece, e Q2 mostra no nó core que sem ele o dono não termina. R4 passa porque no nó core a resposta do bitfield costuma chegar depois de o semeador buscar a saída; R3 mostra no navegador o caso em que só a contagem do que foi servido confirma. + ## Decisões tomadas aqui - A regra do buraco mora na dobra (`bindCores`), não só nos leitores: assim o nó core, o nó do navegador e o serviço de contas dobram igual mesmo com cópias diferentes do mesmo escritor, e um bloco depois de um buraco abaixo do corte nunca conta. @@ -148,6 +199,6 @@ Segunda rodada, uma mutação por vez (`node ` com a mudança tirada, d - (c), (d) e (e): decisões mantidas (acima). Riscos que ficam: (c) uma dobra de catálogo vê menos de um CW cortado do que a dobra completa (só o prefixo), nunca mais; (d) um escritor esparso perto da cota de um leitor conta pelo comprimento assinado, não pelo que o leitor guarda; (e) a banda do semeador para um escritor cortado longo continua a do escritor inteiro até `MAX_LOG_BLOCKS`. - (f) Não feito: parado para decisão (pedido da rodada). As sincronizações (core e navegador) ainda recusam com `header-invalid` o cabeçalho sem caixa num grupo privado antes de guardá-lo, então um leitor que nunca o teve fica com a dobra `incomplete`, embora o semeador mantenha o toco. Fazer exige que o veredito do portão dependa da caminhada do diário que liga o escritor (o registro de corte 0, não só o bloco de entrada que o I1 admite), levantar a recusa da rodada (`refused`) quando essa caminhada muda, guardar só `[0, 1)`, pular esse escritor em `recheckGroupBoxes`, na revalidação da partida e na meta de conclusão, e decidir o que acontece quando a liberação some (registro de corte apagado). Isso muda as regras de admissão (I1 a I4) nas duas sincronizações. - (g) Encontrado na segunda rodada, anterior a este PR (vem da #713): o semeador lista um escritor que o diário desligou com `group.leave` com o seq desse `group-join` `on: false`, e o portão do leitor exige `on: true` naquele seq, então recusa como `unbound` (no navegador, visto no teste: `group-skipped` com `reason: 'unbound'`; no nó core pela leitura do código, `joinVerdict`), cobra como dica ruim do semeador e nunca guarda o escritor. Com o diário guardado, a caminhada espera o escritor e a dobra pode ficar `incomplete`. Não mexido aqui. -- O "respondeu" de `advertisedHeld` é uma aproximação: uma página de bitfield criada por outra mensagem (um `range` ou um `bitfield` do par) antes da resposta ao pedido faz o par contar como respondido cedo. O efeito é um `--once` terminar antes de um bloco anunciado chegar: a dobra trata a cópia como o prefixo (o que está depois do buraco e não chegou não conta) até a próxima sincronização, sem erro. +- O "respondeu" de `advertisedHeld` é uma aproximação: a primeira página de um segmento criada por outra mensagem do par (o `range` de um bloco novo, por exemplo) antes da resposta ao pedido faz o par contar como respondido cedo. A faixa contígua do par não cria página, então isso só acontece com um bloco anunciado avulso nos primeiros 32768 blocos do segmento. O efeito é um `--once` terminar antes de um bloco anunciado chegar: a dobra trata a cópia como o prefixo (o que está depois do buraco e não chegou não conta) até a próxima sincronização, sem erro. +- Um semeador que só tem pares esparsos de um escritor cortado guarda o que eles têm depois do corte; uma saída ou um bloco de chave que nenhum par conectado tem só chega quando um par mais completo aparece (a faixa incerta é relida 1 s depois de um par entrar e, sem isso, a cada até 5 minutos). - No nó core, os cortes do próprio diário são lidos uma vez, quando a sincronização abre os escritores: um corte escrito durante a mesma execução só vale na próxima. -- O dono dá o escritor por enviado no corte; as saídas e os blocos de chave depois do corte podem chegar ao semeador só depois que o `--once` do dono terminou (o semeador os busca quando o dono aparece de novo). From 48861205a8ae0d8a1da89966749614b75579309f Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 03:37:21 -0300 Subject: [PATCH 19/25] fix(seeder): end the wait for a chunk past a cut on a drop or a close A drop destroys the chunk's range, whose done() never settles, and wakes only a wait already armed, so one landing while the loop checked what its peers advertise parked it for good with a peer-add listener on a closed core. The loop now checks for a drop or a close after each await. --- packages/core/lib/seeder.js | 11 +++++-- packages/core/test/seeder-groups-cuts.test.js | 30 ++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/packages/core/lib/seeder.js b/packages/core/lib/seeder.js index 1d73820c..2528cecb 100644 --- a/packages/core/lib/seeder.js +++ b/packages/core/lib/seeder.js @@ -4822,10 +4822,16 @@ class Seeder { p = q } const core = entry.core + // A drop or a close destroys the range, whose done() then never settles, + // and wakes only a wait already armed: one landing during an await here + // would park this loop for good, listening on a closed core. + const gone = () => entry.dropped || this._closing let delay = CUT_POLL_MIN_MS let held = null - while (!p.done && !p.failed) { - if (await advertisedHeld(core, to, from)) { + while (!p.done && !p.failed && !gone()) { + const all = await advertisedHeld(core, to, from) + if (gone()) break + if (all) { held = (await core.has(from, to)) ? 'done' : 'sparse' break } @@ -4845,6 +4851,7 @@ class Seeder { if (why === 'wake') break delay = why === 'tick' ? Math.min(delay * 2, CUT_POLL_MAX_MS) : CUT_POLL_MIN_MS } + if (gone()) return null if (p.done) held = 'done' if (held === null) return null // Its blocks no peer advertised would otherwise come later, past the diff --git a/packages/core/test/seeder-groups-cuts.test.js b/packages/core/test/seeder-groups-cuts.test.js index 9ff7b640..21897ae2 100644 --- a/packages/core/test/seeder-groups-cuts.test.js +++ b/packages/core/test/seeder-groups-cuts.test.js @@ -10,7 +10,7 @@ const test = require('brittle') const b4a = require('b4a') const { testnet, until, localSwarmOptions } = require('./helpers/net') const { createSwarm, attachAdmission } = require('../lib/net') -const { liveGroupJoins, groupWalkStep, groupWalkCut } = require('../lib/seeder') +const { liveGroupJoins, groupWalkStep, groupWalkCut, Seeder } = require('../lib/seeder') const { groupJoinRecord, groupCutRecord, @@ -636,3 +636,31 @@ test('a seeder fed only by a seeder that keeps a writer sparse reads past cut + t.absent('unsureFrom' in readGroups(second.dataDir)[W], 'and the range is listed as read whole') t.is(one.events.filter((e) => e.event === 'fatal').length + two.events.filter((e) => e.event === 'fatal').length, 0) }) + +// A drop or a close destroys the range of a chunk past the cut, whose done() +// then never settles, and wakes only a wait already armed: one landing while +// the loop checks what its peers advertise must still end it. +test('a drop during the check of a chunk past the cut ends its wait (#350)', async (t) => { + const listeners = new Set() + const entry = { past: null, wake: null, dropped: false } + const peer = { remoteSynced: true, remoteContiguousLength: 300, remoteLength: 300, remoteBitfield: { getBitfield: () => null, get: () => false } } + const core = { + peers: [peer], + contiguousLength: 3, + download: () => ({ done: () => new Promise(() => {}), destroy () {} }), + // The drop lands while advertisedHeld reads what is held, and the close + // takes the peers with it. + async has () { + entry.dropped = true + core.peers = [] + return false + }, + on (name, fn) { listeners.add(fn) }, + off (name, fn) { listeners.delete(fn) } + } + entry.core = core + const read = Seeder.prototype._groupPastRange.call({ _closing: false }, entry, 3, 259) + const ended = await Promise.race([read.then((value) => ({ value })), new Promise((resolve) => setTimeout(() => resolve(null), 2000))]) + t.alike(ended, { value: null }, 'the wait ends with nothing read') + t.is(listeners.size, 0, 'and listens on the core no more') +}) From e483d6e0bb2c3591c11d3f5776433b2852950f37 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 03:37:38 -0300 Subject: [PATCH 20/25] test(sparse): cover a block served to the peer in peerHolds No test failed when peerHolds stopped counting the blocks this core served to the peer, which the core node's owner check can need. --- packages/core/test/sparse-held.test.js | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/core/test/sparse-held.test.js b/packages/core/test/sparse-held.test.js index f9751da1..75a6d1fa 100644 --- a/packages/core/test/sparse-held.test.js +++ b/packages/core/test/sparse-held.test.js @@ -9,7 +9,7 @@ const b4a = require('b4a') const Hypercore = require('hypercore') const path = require('node:path') const { tmpDir } = require('./helpers/tmp') -const { advertisedHeld, nextHeld } = require('../lib/sparse-held') +const { advertisedHeld, nextHeld, peerHolds } = require('../lib/sparse-held') const { readHeldBlocks } = require('../lib/store') function link (a, b) { @@ -85,3 +85,15 @@ test('a held block past a hole is found without asking has() for every block (#3 t.alike(entries.map((e) => e.seq), [...Array(10).keys(), 35000], 'the prefix and the leave past the hole') t.ok(calls < 100, 'has() asked ' + calls + ' times, not once per block') }) + +// The owner of a cut writer counts a block past the cut as at the seeder +// once the seeder's peer advertises it or this core served it to that peer: +// a peer never announces a block it gets to one that holds it already. +test('a peer holds a block it advertises or was served (#350)', (t) => { + const peer = { remoteContiguousLength: 3, remoteBitfield: { get: (i) => i === 5 } } + t.ok(peerHolds(peer, [0, 2, 5]), 'its contiguous range and a block it advertises') + t.absent(peerHolds(peer, [5, 7]), 'not a block it neither advertises nor was served') + t.absent(peerHolds(peer, [5, 7], new Set()), 'nor with nothing served') + t.ok(peerHolds(peer, [5, 7], new Set([7])), 'a block served to it counts') + t.absent(peerHolds(peer, [5, 7, 9], new Set([7])), 'only that one') +}) From 6cad413a249a1f90574894a17164501908839ba1 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 03:37:55 -0300 Subject: [PATCH 21/25] fix(seeder): read an unsure range again only for a peer that offers a block there Every peer joining a cut writer reset the backoff to 1 s and read the whole unsure range again, so on a busy seeder each reader that connected, or a peer reconnecting in a loop, drove a full pass for a writer whose device is gone. A peer that joins now reads it again at once only when, once synced and asked for its bitfield there, it advertises a block in the range the seeder lacks; any other peer leaves the backoff. --- packages/core/lib/seeder.js | 43 ++++++++++++++++--- packages/core/lib/sparse-held.js | 11 ++++- packages/core/test/seeder-groups-cuts.test.js | 28 +++++++++++- packages/core/test/sparse-held.test.js | 22 +++++++++- 4 files changed, 93 insertions(+), 11 deletions(-) diff --git a/packages/core/lib/seeder.js b/packages/core/lib/seeder.js index 2528cecb..3dae9c77 100644 --- a/packages/core/lib/seeder.js +++ b/packages/core/lib/seeder.js @@ -229,7 +229,7 @@ const { const { validateMediaHeader, latestMediaCores } = require('./media') const { validateGroupHeader, validateWriterHeader, validateGroupRecord, groupJoins, slotWalkState, slotWalkStep } = require('./group') const { createGroupIndex } = require('./group-index') -const { advertisedHeld } = require('./sparse-held') +const { advertisedHeld, peerAnswered, peerOffers, askHeld } = require('./sparse-held') const { attachGroups } = require('./groups-net') const { presentState, presentLength, clearDeleted, downloadExcept, downloadExceptPages, pageTargets, deletedRanges, shownMedia } = require('./deletion') const { createEngagementIndex } = require('./engagement-index') @@ -991,10 +991,13 @@ const CUT_CHUNK = 256 const CUT_POLL_MIN_MS = 100 const CUT_POLL_MAX_MS = 2000 // A range past a cut that sparse peers only fed (binding unsureFrom, -// unsureTo) is read again (_cutGroupCore): REREAD_PEER_MS after a peer -// joins, so that it has synced, and at intervals growing from -// REREAD_MIN_MS to REREAD_MAX_MS, until a pass reads each chunk whole. +// unsureTo) is read again (_cutGroupCore) at intervals growing from +// REREAD_MIN_MS to REREAD_MAX_MS, until a pass reads each chunk whole, and +// at once when a peer that joins advertises a block there this seeder +// lacks (_rereadIfOffered): checked every REREAD_PEER_MS from its join, so +// that it has synced and answered, up to REREAD_PEER_TRIES times. const REREAD_PEER_MS = 1000 +const REREAD_PEER_TRIES = 10 const REREAD_MIN_MS = 1000 const REREAD_MAX_MS = 5 * 60 * 1000 @@ -4733,10 +4736,10 @@ class Seeder { if (entry.cut !== null && index < keptBelow(entry.cut) && core.byteLength > this.maxLogBytes) this._cutGroupCore(key, entry) } // A peer that joins may hold what sparse peers did not. - const onPeer = () => { + const onPeer = (peer) => { if (entry.dropped || this._closing) return - entry.rereadDelay = REREAD_MIN_MS - this._armReread(key, entry, REREAD_PEER_MS) + const b = this._groups.get(key) + if (b && Number.isSafeInteger(b.unsureFrom)) this._rereadIfOffered(key, entry, peer, REREAD_PEER_TRIES) } core.on('append', checkQuota) core.on('download', onDownload) @@ -4865,6 +4868,32 @@ class Seeder { return held } + // Reads writer K's unsure range again at once when `peer`, which joined, + // advertises a block there this seeder lacks, checked REREAD_PEER_MS from + // now, once it has synced and answered for the range (asked for its + // bitfield there, as a seeder holding the blocks below the cut sends no + // want past them), up to `tries` times. Any other peer, a reader that + // connects or one reconnecting in a loop, leaves the backoff as it is: a + // writer whose device is gone keeps its unsure range for good. + _rereadIfOffered (key, entry, peer, tries) { + const stale = () => entry.dropped || this._closing || this._groupServed.get(key) !== entry || peer.removed === true + const timer = setTimeout(async () => { + try { + const b = this._groups.get(key) + if (stale() || !b || !Number.isSafeInteger(b.unsureFrom)) return + if (peer.remoteSynced === true) askHeld(peer, [b.unsureFrom]) + if (!peerAnswered(peer, b.unsureTo, b.unsureFrom)) { + if (tries > 1) this._rereadIfOffered(key, entry, peer, tries - 1) + return + } + if (!(await peerOffers(entry.core, peer, b.unsureFrom, b.unsureTo)) || stale()) return + entry.rereadDelay = REREAD_MIN_MS + this._armReread(key, entry, 0) + } catch {} + }, REREAD_PEER_MS) + if (timer.unref) timer.unref() + } + // Arms the next read again of writer K's unsure range in `delay` ms (none // without one), replacing any armed before. _armReread (key, entry, delay) { diff --git a/packages/core/lib/sparse-held.js b/packages/core/lib/sparse-held.js index 77bf6ad9..1660b8eb 100644 --- a/packages/core/lib/sparse-held.js +++ b/packages/core/lib/sparse-held.js @@ -102,6 +102,15 @@ async function nextHeld (core, from, end) { return -1 } +// Whether `peer` advertises a block of [from, to) `core` does not hold, one +// this core could still fetch from it. +async function peerOffers (core, peer, from, to) { + for (let i = nextAdvertised(peer, from, to); i !== -1; i = nextAdvertised(peer, i + 1, to)) { + if (!(await core.has(i))) return true + } + return false +} + // Whether `peer` holds every block in `seqs`: it advertises it, or this // core served it to that peer (`served`, a Set of indexes, or null). A // peer never announces a block it gets to one that holds it already, so a @@ -182,4 +191,4 @@ async function heldRuns (core, from, to, max) { return runs } -module.exports = { advertisedHeld, peerAnswered, nextHeld, heldRuns, peerHolds, askHeld, keptPastSeqs, BITFIELD_SEGMENT } +module.exports = { advertisedHeld, peerAnswered, peerOffers, nextHeld, heldRuns, peerHolds, askHeld, keptPastSeqs, BITFIELD_SEGMENT } diff --git a/packages/core/test/seeder-groups-cuts.test.js b/packages/core/test/seeder-groups-cuts.test.js index 21897ae2..5771df7d 100644 --- a/packages/core/test/seeder-groups-cuts.test.js +++ b/packages/core/test/seeder-groups-cuts.test.js @@ -621,8 +621,32 @@ test('a seeder fed only by a seeder that keeps a writer sparse reads past cut + const listed = readGroups(second.dataDir)[W] t.alike([listed.unsureFrom, listed.unsureTo], [3, length], 'the range read from sparse peers only is listed as such') - // B comes back with W whole: the range is read again, each chunk whole, - // and what is not kept is cleared again. + // A peer that joins with nothing there the second seeder lacks (A opens + // W, empty) reads nothing again and leaves the backoff. The backoff is + // pushed out first, so that only a peer could start a pass. + const far = 5 * 60 * 1000 + e2.rereadDelay = far + two.seeder._armReread(W, e2, far) + await until(() => !e2.cutting && !e2.reread, { name: 'nenhuma releitura em curso' }) + let passes = 0 + const pastRange = two.seeder._groupPastRange + two.seeder._groupPastRange = function (...args) { + passes++ + return pastRange.apply(this, args) + } + let joined = false + e2.core.once('peer-add', () => { joined = true }) + const empty = A.store.get({ key: b4a.from(W, 'hex') }) + await empty.ready() + t.teardown(() => empty.close(), { force: true }) + await until(() => joined, { name: 'A ligado a W' }) + await new Promise((resolve) => setTimeout(resolve, 3000)) + t.is(passes, 0, 'a peer offering nothing there starts no pass') + t.is(e2.rereadDelay, far, 'and leaves the backoff') + delete two.seeder._groupPastRange + + // B comes back with W whole, which it advertises: the range is read again + // at once, each chunk whole, and what is not kept is cleared again. const back = createSwarm(localSwarmOptions(bootstrap)) t.teardown(() => back.destroy(), { force: true }) back.on('connection', (socket) => { diff --git a/packages/core/test/sparse-held.test.js b/packages/core/test/sparse-held.test.js index 75a6d1fa..666e5156 100644 --- a/packages/core/test/sparse-held.test.js +++ b/packages/core/test/sparse-held.test.js @@ -9,7 +9,7 @@ const b4a = require('b4a') const Hypercore = require('hypercore') const path = require('node:path') const { tmpDir } = require('./helpers/tmp') -const { advertisedHeld, nextHeld, peerHolds } = require('../lib/sparse-held') +const { advertisedHeld, nextHeld, peerHolds, peerOffers } = require('../lib/sparse-held') const { readHeldBlocks } = require('../lib/store') function link (a, b) { @@ -97,3 +97,23 @@ test('a peer holds a block it advertises or was served (#350)', (t) => { t.ok(peerHolds(peer, [5, 7], new Set([7])), 'a block served to it counts') t.absent(peerHolds(peer, [5, 7, 9], new Set([7])), 'only that one') }) + +// A seeder reads its unsure range again for a peer that joins only when it +// advertises a block there the seeder lacks. +test('a peer offers a block it advertises and the core lacks (#350)', async (t) => { + const held = new Set([4, 6]) + const core = { has: async (i) => held.has(i) } + const firstOf = (set, from) => { + const next = [...set].filter((i) => i >= from).sort((a, b) => a - b) + return next.length > 0 ? next[0] : -1 + } + const peer = (contiguous, set) => ({ + remoteContiguousLength: contiguous, + remoteBitfield: { get: (i) => set.has(i), firstSet: (from) => firstOf(set, from) } + }) + t.absent(await peerOffers(core, peer(0, new Set()), 3, 10), 'nothing advertised') + t.absent(await peerOffers(core, peer(0, new Set([4, 6])), 3, 10), 'only what the core holds') + t.ok(await peerOffers(core, peer(0, new Set([4, 8])), 3, 10), 'a block the core lacks') + t.absent(await peerOffers(core, peer(0, new Set([4, 12])), 3, 10), 'not past the range') + t.ok(await peerOffers(core, peer(10, new Set()), 3, 10), 'its contiguous range over a block the core lacks') +}) From 95b4aa850c1cde2993631acb1215b6161c22e919 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 03:38:48 -0300 Subject: [PATCH 22/25] docs: describe the peer-offer reread and the fourth round's proofs Protocol: a peer that joins reads an unsure range again at once only when it advertises a block there the seeder lacks. The verification file adds the fourth round's changes, runs and mutation proofs, and names the single-block announcement among the answered approximation's causes. --- docs/protocol.md | 2 +- docs/verification/feat-writer-slot-sparse.md | 34 +++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/docs/protocol.md b/docs/protocol.md index 0f685abd..f3c75ba6 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -2138,7 +2138,7 @@ Mirrors "Seeder media admission" (section 5) with the same invariants, plus the - **Root G.** Admitted by the same steps as a writer (including quarantine, persistence before the open and the held blocklist) when its block 0 is a valid `group-header` and some served, gated, unblocked diary has a live join for G (the creator's join with `writer == group` is the normal path). **Root binding.** The header's diary counts as the root's creator (and as support) only when that diary is served, not dropped, not blocked and has its own `on: true` `group-join` for writer = G. Otherwise the root is supported only by real member joins, and `/index/group/` answers `creator: null` (also during the startup window before the creator's scan). **Support:** a served root stays while at least one served, unblocked diary keeps it: the creator if it bound the root, a diary with a bound writer in it, or a live join. Withdrawing every join does not drop a root that still has a bound writer. At startup a root does not survive just because its creator's diary is kept: decisions that find a root unsupported are deferred until the startup scans of the served diaries end, and it is judged after them; an unsupported root is then dropped with reason `unbound`. **Blocked or revoked creator:** a served root is purged for a blocked diary, or one whose certificate issuer is revoked, only when that diary BOUND the root (`bound: true`: its header diary has its own `on: true` join for writer = G). Reason `blocked` (also during the startup window), the root stays refused at admission and at startup while its creator stays gone, even while members support the group. A root whose header merely names a blocked diary that never bound it has no creator and stays while members support it. A creator that is both blocked and from a revoked issuer has that issuer recorded too (`revokedCreators`, and `goneRoots` for its bound roots), and a stored issuer is never replaced by null: lifting only the block keeps its bound roots refused, also after a restart, while the issuer stays revoked. A creator dropped or refused for a revoked issuer is remembered with that issuer (`revokedCreators`, a moderation outcome, also across restarts) and judged against the CURRENT revoked-issuer set: adopting a list that lifts the revocation lifts it. An announce of a revoked-issuer certificate is not remembered at all, not even in memory: only a diary's own block-0 verdict (refused at startup, or dropped as revoked-issuer while served) feeds `revokedCreators`, a `goneRoots` issuer and the refusal of a root at admission for a revoked creator, so a root whose creator was never served on this seeder is not refused for a revoked issuer (the root only names that diary). A repeated revoked announce of the same key from one peer counts as a bad announce; the first does not. A purged root is remembered with its creator and, for a revocation, its issuer (`goneRoots`) and refused again while that creator stays gone; lifting the block or revocation lets the same join admit it, and a root admitted again after that keeps `bound`. A diary admitted with a verifying certificate clears its stale revoked entries (`revokedCreators` and the `goneRoots` entries naming it with an issuer). A root whose header names a currently blocked diary, or a diary whose known issuer is revoked, is refused at admission even without a purge record (a root admitted before the block stays until its binding says otherwise). Roots bound before this version get `bound` only after their creator's next scan. A root its creator bound stays listed on `/index/group/` (and in the writer pages) under that creator diary while members keep it served after the creator is dropped for a reason other than moderation (quota, fork, seeder full), also after a restart; `header.diary` stays `null` then, so no creator authority is shown. Members' own writers stay under the rules above. **Issuer kept with the binding.** A bound root's binding keeps the issuer of its creator's block-0 certificate (`issuer`, taken at the creator's scan). When that issuer is revoked under the current list and the creator is not served, the bound root is purged with reason `revoked-issuer` and remembered like any revoked-issuer drop (`revokedCreators`, `goneRoots`), also at startup and also when the creator left first for another reason, such as its quota, so a member that keeps the root served cannot outlast the revocation. **Header fields kept.** While any core of group G is served or being admitted, the seeder keeps the verified header fields of its root (`kind`, `visibility`, `join`, `community`, `member`, and `name` for a public root) in `groupHeaders`, also after the root was purged or refused or its block 0 is withheld, and drops them once no core of G is left or G is blocked; past the bound the record is not kept. A persisted record without `community` and `member` is a protocol v1 leftover and is dropped at start. - **Protocol v2 admission and scan.** A header whose validator returns `legacy` is `header-invalid` (no new wire reason), and the startup re-gate purges every stored v1 root and writer with that reason. A group root must carry a hex64 `community`; the seeder never checks a writer's `member` or any authorization (the fold does). CWs are admitted like any writer and take part in the writer pages. For a served community root the index folds `group-meta`, `group-admin` and `group-create` from the root only (creates are indexed from C's root alone; a CW is not scanned for them, `group-create` being `root-only` there). A public group root is scanned for `group-meta` only (one scan at a time across group roots, skipping blocks without the bytes `group-meta`) and a private root is not scanned. **Denial bounds.** `revokedCreators` and `goneRoots` never evict an entry still in force: when a map reaches 4096 entries the lifted ones go first, and `admitted.json` may hold up to 8192 entries in each. While 4096 or more denials are in force in either map, or after a denial found no room (past 8192), the seeder refuses every group root with the group-rejected reason `denials-full` and logs `{ event: 'warning', reason: 'denials-full' }`, so a purged root cannot return for lack of memory. -- **Cut writers** (#350, owner decision). A writer its diary cut (frozen, released, or live with a cut once the frozen slots or the releases are spent: `groupWalkCut` over the slot walk) keeps its blocks below the cut, block 0 always, and from the cut on only its `group-leave` and `group-key` blocks, the ones the fold still counts there and readers read past a hole ("Diary cuts on own writers", "Sparse copies"). A released writer (cut 0) is a stub: block 0, its leaves and its key blocks. Its download stops at the cut; the blocks past it, up to `MAX_LOG_BLOCKS`, are fetched `CUT_CHUNK` = 256 at a time, read for their type only (`decodeBlock`, never validated) and cleared unless kept. Hypercore keeps the tree nodes the proofs of the kept blocks need, so a reader verifies each against the signed length. The binding in admitted.json carries `cut`, `clearedFrom` and `clearedTo` (the range past the cut already read) and is persisted before a chunk is cleared; the next serve reads that range again and clears what it still holds beyond the kept blocks, so a crash in between leaves nothing extra for good. A chunk ends once every block its synced peers advertise there is held (`advertisedHeld`, "Completion goals"), even when they lack some of it: a seeder fed only by peers that keep the writer sparse (another seeder, or after a `quota` drop that a lower cut lifted) still reads past the first chunk. What it holds there is kept or cleared as above, and a chunk read that way joins the binding's unsure range (`unsureFrom`, `unsureTo`, within the range already read), read again chunk by chunk 1 s after a peer joins the core and at intervals doubling from 1 s to 5 minutes; a chunk read again whole leaves it, so a leave or key block those peers lacked comes from a fuller peer. The quota counts what is kept: at admission and at startup, the blocks below the cut (`min(max(cut, 1), length)` against `maxLogBlocks`); while the whole log is over the quota, the blocks below the cut and those kept past it, and their bytes, against `maxLogBlocks` and `maxLogBytes`, counted as they are read. A writer past it is dropped with `quota` and its join is refused as at admission. The caps above count a stub like any writer. The diary's scan gives the cut: a lower one applies at once (also to a core still opening), a higher one or none only once the scan caught up with the diary (a cut restored from admitted.json whose record was deleted, for example), and then the download reaches the new cut, or the whole log, so what clearing removed is fetched again. A cut restored at startup clears nothing new until the scan confirms it. A join refused before (`quota`, `header-invalid`) is tried again once per lower cut. A writer of a private group whose header has no box (the one a node replaced, "Writers named by a node") is `header-invalid` while counted and kept as a stub once released, so its block 0 stays available to readers, which reject it by its header ("Limits"); if its release goes away it is judged again and dropped. +- **Cut writers** (#350, owner decision). A writer its diary cut (frozen, released, or live with a cut once the frozen slots or the releases are spent: `groupWalkCut` over the slot walk) keeps its blocks below the cut, block 0 always, and from the cut on only its `group-leave` and `group-key` blocks, the ones the fold still counts there and readers read past a hole ("Diary cuts on own writers", "Sparse copies"). A released writer (cut 0) is a stub: block 0, its leaves and its key blocks. Its download stops at the cut; the blocks past it, up to `MAX_LOG_BLOCKS`, are fetched `CUT_CHUNK` = 256 at a time, read for their type only (`decodeBlock`, never validated) and cleared unless kept. Hypercore keeps the tree nodes the proofs of the kept blocks need, so a reader verifies each against the signed length. The binding in admitted.json carries `cut`, `clearedFrom` and `clearedTo` (the range past the cut already read) and is persisted before a chunk is cleared; the next serve reads that range again and clears what it still holds beyond the kept blocks, so a crash in between leaves nothing extra for good. A chunk ends once every block its synced peers advertise there is held (`advertisedHeld`, "Completion goals"), even when they lack some of it: a seeder fed only by peers that keep the writer sparse (another seeder, or after a `quota` drop that a lower cut lifted) still reads past the first chunk. What it holds there is kept or cleared as above, and a chunk read that way joins the binding's unsure range (`unsureFrom`, `unsureTo`, within the range already read), read again chunk by chunk at intervals doubling from 1 s to 5 minutes, and at once when a peer that joins the core advertises a block there the seeder lacks (checked every second from its join, up to 10 times, once it has synced and answered the want the seeder sends it for that range); a peer offering nothing there, such as a reader that connects or one reconnecting in a loop, leaves the backoff as it is. A chunk read again whole leaves the range, so a leave or key block those peers lacked comes from a fuller peer. The quota counts what is kept: at admission and at startup, the blocks below the cut (`min(max(cut, 1), length)` against `maxLogBlocks`); while the whole log is over the quota, the blocks below the cut and those kept past it, and their bytes, against `maxLogBlocks` and `maxLogBytes`, counted as they are read. A writer past it is dropped with `quota` and its join is refused as at admission. The caps above count a stub like any writer. The diary's scan gives the cut: a lower one applies at once (also to a core still opening), a higher one or none only once the scan caught up with the diary (a cut restored from admitted.json whose record was deleted, for example), and then the download reaches the new cut, or the whole log, so what clearing removed is fetched again. A cut restored at startup clears nothing new until the scan confirms it. A join refused before (`quota`, `header-invalid`) is tried again once per lower cut. A writer of a private group whose header has no box (the one a node replaced, "Writers named by a node") is `header-invalid` while counted and kept as a stub once released, so its block 0 stays available to readers, which reject it by its header ("Limits"); if its release goes away it is judged again and dropped. - **Drops** (purge before un-listing, runtime order of media (e), each logs `{ event: 'group-rejected', key, diary, group, reason }`): the diary is dropped or blocked (D's writers go with it, with `blocked` or the diary's own reason; the root stays while supported), the root key is in the blocklist snapshot (`group-blocked`, for the root and every writer of that group; the blocklist apply loop also walks the group map, because it otherwise walks only `_served`), a fork (`fork`), the quota (`quota`), an unsupported root (`unbound`), a root whose creator (the header diary, when it bound the root) is blocked or has a revoked issuer (`blocked`) or a failed header re-check (`header-invalid`). NEVER on a `group-leave`, never on a join turned off and never on a cut: a cut or released writer stays held whole ("Cut writers"). - **Retention.** Index withdrawal and membership end are distinct, as defined above. Admitted writer and root cores stay served and listed across either event (the index only marks the writer `left`), preserving the moderation records of departed members. - **Startup** re-gates listed group cores like listed media cores (a core whose binding is gone, or whose root is blocked, is purged first), applying the same per-diary counts to persisted bindings (32 per group); for bindings written before the cap that count follows iteration order, not earliest join. A cut writer, a released one included, is restored whole (quota on the whole log), and the scans recompute cuts and releases from the diaries. `seed list` prints the group cores indented under their diary (`grupo (grupo ) blocos bytes`), `seed drop` purges them as auxiliaries, and `stats` adds `groupCores`. diff --git a/docs/verification/feat-writer-slot-sparse.md b/docs/verification/feat-writer-slot-sparse.md index 6f6280ac..2f18cb34 100644 --- a/docs/verification/feat-writer-slot-sparse.md +++ b/docs/verification/feat-writer-slot-sparse.md @@ -37,7 +37,7 @@ Documentos: `docs/protocol.md` (bit `CAP_WRITER_CUTS`, entrada de `buildGroup` e Terceira rodada (revisão em `d487e5e7`, sem P1): -- Semeador, pedaço depois do corte lido de pares esparsos (P2-1, `_groupPastRange` e `_cutGroupCore` em `seeder.js`): um pedaço termina quando todo bloco que os pares sincronizados anunciam ali está guardado (`advertisedHeld`), não só quando os 256 blocos chegaram. O que ele guarda ali é mantido (e contado na cota) ou limpo como antes, e o pedaço entra na faixa incerta da ligação (`unsureFrom`, `unsureTo`, dentro da faixa já lida), relida pedaço a pedaço 1 s depois de um par entrar no núcleo e em intervalos que dobram de 1 s a 5 minutos; um pedaço relido inteiro sai dela. Antes, um semeador que só recebia o escritor de pares esparsos (outro semeador, ou depois de uma saída por `quota` que um corte menor levantou) nunca passava do primeiro pedaço, e uma saída em corte + 256 ou depois nunca era buscada. +- Semeador, pedaço depois do corte lido de pares esparsos (P2-1, `_groupPastRange` e `_cutGroupCore` em `seeder.js`): um pedaço termina quando todo bloco que os pares sincronizados anunciam ali está guardado (`advertisedHeld`), não só quando os 256 blocos chegaram. O que ele guarda ali é mantido (e contado na cota) ou limpo como antes, e o pedaço entra na faixa incerta da ligação (`unsureFrom`, `unsureTo`, dentro da faixa já lida), relida pedaço a pedaço em intervalos que dobram de 1 s a 5 minutos (e, desde a quarta rodada, na hora só quando um par que entra anuncia ali um bloco que falta); um pedaço relido inteiro sai dela. Antes, um semeador que só recebia o escritor de pares esparsos (outro semeador, ou depois de uma saída por `quota` que um corte menor levantou) nunca passava do primeiro pedaço, e uma saída em corte + 256 ou depois nunca era buscada. - Dono de um escritor cortado (P2-2): o nó core (`ownKeptMissing` em `sync.js`) e o navegador (`seederHoldsKept` em `checkMedia`) só dão o escritor por enviado quando, além de o semeador guardá-lo até `max(corte, 1)`, o par de replicação do semeador anuncia cada `group-leave` e `group-key` próprio do corte em diante, ou recebeu esse bloco nesta conexão (evento `upload`: um par nunca anuncia um bloco que recebe a quem já o tem). `askHeld` (`sparse-held.js`) pede a esse par o bitfield ali, porque uma cópia inteira não manda pedidos próprios. Enquanto falta, o nó core responde `:corte ` e o navegador `:push `, sem `group-pushed`. - `peerAnswered` (P3-3): um pedido pede um segmento inteiro do bitfield (2097152 blocos), e a resposta traz a primeira página do segmento e as páginas seguintes que o par tem. O par conta como respondido quando chegou a primeira página de cada segmento do intervalo; antes era preciso toda página de 32768 bits até o fim, e um par esparso num escritor longo nunca respondia. - `readHeldBlocks` (P3-6, `store.js` e `node-store.js`): depois do buraco, pula para o próximo bloco guardado pelo bitfield que o próprio `has()` lê (`firstSet`, presente no Hypercore 10 e no 11, conferido em `node_modules`), em vez de um `has()` por seq. Sessão que não seja a padrão (snapshot, batch) continua com o `has()` por bloco. @@ -45,6 +45,12 @@ Terceira rodada (revisão em `d487e5e7`, sem P1): - `sparseMet` do navegador (P3-4): a espera entre conferências dobra até 5 s (`SPARSE_CHECK_MAX_MS`) enquanto a faixa guardada da cópia não muda; um download ou um par que entra ou sai volta a 100 ms. Antes eram 100 ms enquanto a aba ficasse aberta. - Comentários (P3-7, `seeder.js`): `released = (cut === 0)` cobre também o escritor ainda contado cortado em 0 depois de gastas as 16 liberações; os dois ficam como toco. +Quarta rodada (revisão de `d487e5e7..9b2137f8`, sem P1 nem P2): + +- `_groupPastRange` (P3-b): o laço confere `entry.dropped || this._closing` depois de cada `await` e sai com `null`. Antes, uma queda durante o `await advertisedHeld` deixava o laço parado para sempre, com um ouvinte `peer-add` num núcleo fechado: o `range.destroy()` do Hypercore não rejeita o `done()`, e o `stopQuota` só acorda uma espera já armada. +- Releitura da faixa incerta quando um par entra (P3-a, `_rereadIfOffered` em `seeder.js`, `peerOffers` em `sparse-held.js`): só relê na hora quando o par que entrou, depois de sincronizar e responder ao pedido do bitfield dessa faixa (`askHeld`), anuncia ali um bloco que o semeador não tem. Conferido a cada 1 s desde a entrada, até 10 vezes. Qualquer outro par (um leitor que conecta, um par que reconecta em laço) deixa o recuo como está. Antes, cada `peer-add` voltava o recuo a 1 s e relia a faixa inteira, então o recuo de 5 minutos nunca valia para um escritor cujo aparelho sumiu. +- `peerHolds` ganhou um teste de unidade com blocos servidos (a mutação R4 da terceira rodada passava). + ## Como verificar Cada arquivo sozinho, a partir da raiz, com `node `, e `npx standard` nos arquivos mudados: @@ -110,6 +116,17 @@ Terceira rodada, cada arquivo sozinho: O teste 15 de `seeder-groups-cuts.test.js` monta o caso P2-1: o primeiro semeador guarda W esparso (corte 3; cinco posts, mais 300, a saída e mais um post, então a saída fica depois de corte + 256); B sai, e um segundo semeador recebe W só do primeiro. Ele guarda a saída, conta-a na cota (`keptPast`) e lista como incerta a faixa lida só de pares esparsos (`unsureFrom` 3, `unsureTo` o comprimento); quando B volta com W inteiro, relê a faixa, limpa de novo o que não fica e tira a faixa incerta da ligação. O teste 2 de `packages/core/test/sync-groups-sparse.test.js` e o teste 3 de `packages/web-member/test/sync-groups-sparse.test.js` seguram `_groupPastRange` do semeador: com a saída só no aparelho, o `--once` do dono não termina `ok` e diz o que falta (nó core `:corte `, navegador `:push ` e nenhum `group-pushed`); solto o semeador, termina `ok`. O caso do `peerAnswered` (`packages/core/test/sparse-held.test.js`, teste 1) usa um escritor de 40000 blocos e um par com `[0, 10)` mais o bloco 500, como o repro da revisão. O pulo pelo bitfield do `readHeldBlocks` tem teste só no nó core (teste 2 do mesmo arquivo); o do navegador são as mesmas quatro linhas, sem teste próprio. O teste da espera do `sparseMet` (P3-4) é de unidade, sobre `sparseCheckDelay`; o empurrão por download ou par não tem teste que falhe sem ele, e os testes esparsos do navegador passam com ele. +Quarta rodada, cada arquivo sozinho com `VARANDA_TEST_CONCURRENCY=1 node `: + +| Arquivo | Testes | Asserções | +|---|---|---| +| `packages/core/test/seeder-groups-cuts.test.js` (teste 15 com o par que nada oferece, teste 16 novo) | 16/16 | 340/340 | +| `packages/core/test/sparse-held.test.js` (testes 3 e 4 novos) | 4/4 | 18/18 | + +`npm run lint` passa. + +No teste 15, antes de B voltar, o recuo do segundo semeador é empurrado para 5 minutos e A abre W vazio: o par novo não oferece nada na faixa incerta, nenhuma passada começa em 3 s e o recuo fica. B volta com W inteiro e a faixa é relida na hora, pelo anúncio de B, não pelo recuo. O teste 16 (novo, de unidade) chama `_groupPastRange` com um núcleo falso cujo `has()` derruba a entrada e leva os pares durante o `advertisedHeld`: a espera tem de terminar com `null` e sem ouvinte no núcleo. Os testes 3 e 4 de `sparse-held.test.js` (novos, de unidade) cobrem `peerHolds` com blocos servidos e `peerOffers`. + Não rodados aqui, por regra do brief: a suíte inteira e a demo (a sessão que abriu esta roda a CI de entrega). ## Prova de regressão @@ -174,7 +191,16 @@ Terceira rodada, uma mutação por vez: | S1 carimbo com o limite antigo de 4096 blocos | `packages/web-member/test/group-memo.test.js` | 11/12, 63/65, falham "signed with 5990 blocks missing past its prefix" e "a block arriving past the hole changes it" | | S2 espera sempre de 100 ms | `packages/web-member/test/sync-groups-sparse.test.js` | 2/3, 25/27, falham "twice the last wait while unchanged" e "up to the bound" | -As linhas R e Q rodaram antes do teste de unidade da S2, por isso 2 testes no arquivo do navegador. R2 passa porque, neste teste, o pedido que `seederHoldsMedia` já faz ao conectar (o segmento 0, antes de chegar a faixa contígua) traz o bitfield depois do corte; `askHeld` cobre o caso em que isso não acontece, e Q2 mostra no nó core que sem ele o dono não termina. R4 passa porque no nó core a resposta do bitfield costuma chegar depois de o semeador buscar a saída; R3 mostra no navegador o caso em que só a contagem do que foi servido confirma. +As linhas R e Q rodaram antes do teste de unidade da S2, por isso 2 testes no arquivo do navegador. R2 passa porque, neste teste, o pedido que `seederHoldsMedia` já faz ao conectar (o segmento 0, antes de chegar a faixa contígua) traz o bitfield depois do corte; `askHeld` cobre o caso em que isso não acontece, e Q2 mostra no nó core que sem ele o dono não termina. R4 passa porque no nó core a resposta do bitfield costuma chegar depois de o semeador buscar a saída; R3 mostra no navegador o caso em que só a contagem do que foi servido confirma, e na quarta rodada o teste de unidade de `peerHolds` (T3, abaixo) falha sem ela. + +Quarta rodada, uma mutação por vez: + +| Mutação | Arquivo | Resultado | +|---|---|---| +| T1 sem conferir a queda depois do `advertisedHeld` | `packages/core/test/seeder-groups-cuts.test.js` | 15/16, 338/340, falham "the wait ends with nothing read" e "and listens on the core no more" (teste 16) | +| T2 cada `peer-add` volta o recuo a 1 s e relê (o código antigo) | o mesmo | 15/16, 338/340, falham "a peer offering nothing there starts no pass" e "and leaves the backoff" (teste 15) | +| T4 um par que oferece um bloco não relê nada | o mesmo | tempo esgotado "W relido inteiro" (teste 15: com o recuo em 5 minutos, só o anúncio de B relê) | +| T3 `peerHolds` sem os blocos servidos (R4) | `packages/core/test/sparse-held.test.js` | 3/4, 17/18, falha "a block served to it counts" | ## Decisões tomadas aqui @@ -199,6 +225,6 @@ As linhas R e Q rodaram antes do teste de unidade da S2, por isso 2 testes no ar - (c), (d) e (e): decisões mantidas (acima). Riscos que ficam: (c) uma dobra de catálogo vê menos de um CW cortado do que a dobra completa (só o prefixo), nunca mais; (d) um escritor esparso perto da cota de um leitor conta pelo comprimento assinado, não pelo que o leitor guarda; (e) a banda do semeador para um escritor cortado longo continua a do escritor inteiro até `MAX_LOG_BLOCKS`. - (f) Não feito: parado para decisão (pedido da rodada). As sincronizações (core e navegador) ainda recusam com `header-invalid` o cabeçalho sem caixa num grupo privado antes de guardá-lo, então um leitor que nunca o teve fica com a dobra `incomplete`, embora o semeador mantenha o toco. Fazer exige que o veredito do portão dependa da caminhada do diário que liga o escritor (o registro de corte 0, não só o bloco de entrada que o I1 admite), levantar a recusa da rodada (`refused`) quando essa caminhada muda, guardar só `[0, 1)`, pular esse escritor em `recheckGroupBoxes`, na revalidação da partida e na meta de conclusão, e decidir o que acontece quando a liberação some (registro de corte apagado). Isso muda as regras de admissão (I1 a I4) nas duas sincronizações. - (g) Encontrado na segunda rodada, anterior a este PR (vem da #713): o semeador lista um escritor que o diário desligou com `group.leave` com o seq desse `group-join` `on: false`, e o portão do leitor exige `on: true` naquele seq, então recusa como `unbound` (no navegador, visto no teste: `group-skipped` com `reason: 'unbound'`; no nó core pela leitura do código, `joinVerdict`), cobra como dica ruim do semeador e nunca guarda o escritor. Com o diário guardado, a caminhada espera o escritor e a dobra pode ficar `incomplete`. Não mexido aqui. -- O "respondeu" de `advertisedHeld` é uma aproximação: a primeira página de um segmento criada por outra mensagem do par (o `range` de um bloco novo, por exemplo) antes da resposta ao pedido faz o par contar como respondido cedo. A faixa contígua do par não cria página, então isso só acontece com um bloco anunciado avulso nos primeiros 32768 blocos do segmento. O efeito é um `--once` terminar antes de um bloco anunciado chegar: a dobra trata a cópia como o prefixo (o que está depois do buraco e não chegou não conta) até a próxima sincronização, sem erro. -- Um semeador que só tem pares esparsos de um escritor cortado guarda o que eles têm depois do corte; uma saída ou um bloco de chave que nenhum par conectado tem só chega quando um par mais completo aparece (a faixa incerta é relida 1 s depois de um par entrar e, sem isso, a cada até 5 minutos). +- O "respondeu" de `advertisedHeld` é uma aproximação: a primeira página de um segmento criada por outra mensagem do par (o anúncio de um bloco só, um `range` de comprimento 1, por exemplo) antes da resposta ao pedido faz o par contar como respondido cedo. A faixa contígua do par não cria página, então isso só acontece com um bloco anunciado avulso nos primeiros 32768 blocos do segmento. O efeito é um `--once` terminar antes de um bloco anunciado chegar: a dobra trata a cópia como o prefixo (o que está depois do buraco e não chegou não conta) até a próxima sincronização, sem erro. +- Um semeador que só tem pares esparsos de um escritor cortado guarda o que eles têm depois do corte; uma saída ou um bloco de chave que nenhum par conectado tem só chega quando um par mais completo aparece (a faixa incerta é relida na hora quando um par que entra anuncia ali um bloco que falta e, sem isso, a cada até 5 minutos). Um par que só tem esse bloco depois da faixa contígua dele conta pela resposta ao pedido do bitfield; se ela não chega em 10 s, vale o recuo. - No nó core, os cortes do próprio diário são lidos uma vez, quando a sincronização abre os escritores: um corte escrito durante a mesma execução só vale na próxima. From 7f6b9722c5f1b36ab225f1c7374428ed01f63072 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 04:19:19 -0300 Subject: [PATCH 23/25] test(seeder): fit the cut tests to the merge of main A deleted group-cut still counts since #717, so a delete no longer makes a cut go: one test now checks that the deleted cut stays, after a restart too, and another drives the dropped-cut path from a stale admitted.json. The stored-binding unit test from #721 gets a scanned diary and a served entry, which _groupCutScan reads on every reconcile. --- packages/core/test/seeder-groups-cuts.test.js | 60 ++++++++++++++++--- 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/packages/core/test/seeder-groups-cuts.test.js b/packages/core/test/seeder-groups-cuts.test.js index fa989f6d..7ea1fbe8 100644 --- a/packages/core/test/seeder-groups-cuts.test.js +++ b/packages/core/test/seeder-groups-cuts.test.js @@ -12,6 +12,7 @@ const test = require('brittle') const b4a = require('b4a') const { testnet, until, localSwarmOptions } = require('./helpers/net') const { createSwarm, attachAdmission } = require('../lib/net') +const { openStore } = require('../lib/store') const { Seeder, liveGroupJoins, groupWalkStep, groupWalkCut } = require('../lib/seeder') const { groupJoinRecord, @@ -141,14 +142,16 @@ test('a stored binding the scan no longer lists gets its on join too, never an o const H = 'bb'.repeat(32) const W = key(0) const { decls, walks } = diary([join(W), join(W, false), join(W, true, H)]) - const entry = { gated: true, dropped: false, groupDecls: decls, groupWalks: walks, groupBound: new Map(), groupRefused: new Map(), groupLive: liveGroupJoins(decls, walks) } + // A diary the scan read to its length (its certificate and three records), + // as _groupCutScan reads it on every reconcile. + const entry = { gated: true, dropped: false, scanned: 4, core: { length: 4 }, groupDecls: decls, groupWalks: walks, groupBound: new Map(), groupRefused: new Map(), groupLive: liveGroupJoins(decls, walks) } t.is(entry.groupLive.get(W).group, H, 'the scan lists W in H only') const rows = [] const seeder = Object.create(Seeder.prototype) Object.assign(seeder, { _closing: false, _groups: new Map([[W, { diary: D, group: G, seq: 2 }]]), - _groupServed: new Map([[W, {}]]), + _groupServed: new Map([[W, { dropped: false, cut: null, cutConfirmed: false }]]), _groupIndex: { addWriter: (group, k, d, seq) => rows.push({ group, key: k, diary: d, seq }), setLeft: (k, left) => rows.push({ key: k, left }) @@ -493,7 +496,7 @@ test('a restart restores a writer at its persisted cut and the scan lowers it or t.is(second.events.filter((e) => e.event === 'fatal').length, 0) }) -test('a cut that is gone after a restart fetches again what clearing removed (#350)', async (t) => { +test('a cut its diary deleted still applies at the seeder, after a restart too (#350)', async (t) => { t.timeout(240000) const { bootstrap } = await testnet(t) const assoc = makeAssociation() @@ -508,14 +511,55 @@ test('a cut that is gone after a restart fetches again what clearing removed (#3 await until(async () => first.seeder._groups.get(W).clearedTo === 6 && !first.seeder._groupServed.get(W).cutting, { name: 'W lido depois do corte' }) t.alike(await blocksHeld(first.seeder._groupServed.get(W).core, 0, 6), [0, 1, 2], 'cleared past its cut') - // B deletes the cut record: the seeder clears that block, so after a - // restart its scan never sees the cut. + // B deletes the cut record: a group-cut ignores deletes (docs/protocol.md + // "Deletion"), so the seeder keeps both the block and the cut. await B.diary.append(deleteRecord({ seq: 2 })) - const diaryCopy = first.seeder._served.get(B.key).core - await until(async () => !(await diaryCopy.has(2)), { name: 'corte apagado no semeador' }) - t.is(first.seeder._groups.get(W).cut, 3, 'the running seeder keeps the cut') + await settledFor(first.seeder, B) + t.ok(await first.seeder._served.get(B.key).core.has(2), 'the deleted cut stays at the seeder') + t.is(first.seeder._groups.get(W).cut, 3, 'and so does the cut') await first.seeder.close() + const second = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex }) + const entry = second.seeder._groupServed.get(W) + t.ok(entry, 'W served at ready') + await settledFor(second.seeder, B) + t.is(entry.cut, 3, 'the scan reads the deleted cut again') + t.alike(readGroups(dataDir)[W], { diary: B.key, group: R, seq: 1, cut: 3, clearedFrom: 3, clearedTo: 6 }, 'the binding keeps it') + t.alike(await blocksHeld(entry.core, 0, 6), [0, 1, 2], 'and nothing it cleared is fetched again') + t.is(first.events.filter((e) => e.event === 'fatal').length + second.events.filter((e) => e.event === 'fatal').length, 0) +}) + +test('a persisted cut the diary does not give is dropped once the scan caught up, and what clearing removed is fetched again (#350)', async (t) => { + t.timeout(240000) + const { bootstrap } = await testnet(t) + const assoc = makeAssociation() + const { dataDir } = prepareSeederDir(t, assoc) + const first = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex }) + + // B's diary: certificate and the join of W, no cut. + const { A, B, R, W } = await cutWorld(t, bootstrap, assoc) + await admit(A, assoc) + await admit(B, assoc) + await admitted(first.events, W) + await downloaded(first.seeder, W, 6) + await first.seeder.close() + + // A cut stored with W's binding and cleared past it, as admitted.json + // keeps one whose record the scan no longer gives (a store restored from + // an older copy, say): a deleted cut still counts since #717, so a + // delete cannot make one go. + const file = path.join(dataDir, 'admitted.json') + const data = JSON.parse(fs.readFileSync(file, 'utf8')) + Object.assign(data.groups[W], { cut: 3, clearedFrom: 3, clearedTo: 6 }) + fs.writeFileSync(file, JSON.stringify(data)) + const store = await openStore(dataDir) + const core = store.get({ key: b4a.from(W, 'hex') }) + await core.ready() + await core.clear(3, 6) + t.alike(await blocksHeld(core, 0, 6), [0, 1, 2], 'cleared past the stored cut') + await core.close() + await store.close() + const second = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex }) const entry = second.seeder._groupServed.get(W) t.ok(entry, 'W served at ready') From 0cc45358db4ead96882c4941f92fbe4c8a2a453c Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 04:21:48 -0300 Subject: [PATCH 24/25] fix(seeder): try a refused join again when a deleted cut arrives late --- packages/core/lib/seeder.js | 6 ++++-- packages/core/test/seeder-delete.test.js | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/packages/core/lib/seeder.js b/packages/core/lib/seeder.js index cce7739e..a1ed8e64 100644 --- a/packages/core/lib/seeder.js +++ b/packages/core/lib/seeder.js @@ -1018,7 +1018,8 @@ function foldGroupRecord (entry, i, r) { } // A lower cut keeps less (the quota) and a release keeps a boxless writer -// (groupVerdict): a join refused before is tried again, once per lower cut. +// (groupVerdict): a join refused before is tried again, once per lower cut, +// whether the scan read the cut or fetched it late as a deleted hole. function foldGroupCut (entry, i, r) { const before = groupWalkCut(entry.groupWalks, r.group, r.writer) foldGroupRecord(entry, i, r) @@ -3739,7 +3740,8 @@ class Seeder { if (r === null || entry.groupSeqs.has(seq)) return this._noteGroupKept(entry, seq) if (r.type !== 'group-join' && r.type !== 'group-cut') return - foldGroupRecord(entry, seq, r) + if (r.type === 'group-cut') foldGroupCut(entry, seq, r) + else foldGroupRecord(entry, seq, r) entry.groupLive = liveGroupJoins(entry.groupDecls, entry.groupWalks) this._reconcileGroups(key, entry) return diff --git a/packages/core/test/seeder-delete.test.js b/packages/core/test/seeder-delete.test.js index cce2dc28..6d093511 100644 --- a/packages/core/test/seeder-delete.test.js +++ b/packages/core/test/seeder-delete.test.js @@ -1414,6 +1414,7 @@ function groupFixture (held, blocks = GROUP_BLOCKS) { groupWalks: new Map(), groupEvents: new Map(), groupLive: new Map(), + groupRefused: new Map(), recent: [], core: { fork: 0, @@ -1480,6 +1481,20 @@ test('a deleted group-cut that arrives after the scan steps the slot walk at its t.alike(entry.groupLive.get(CUT_WRITERS[8]), { group: GROUP_KEY, seq: 10, left: false }, 'the ninth takes the slot it freed') }) +test('a deleted cut that arrives after the scan tries again the join it lowers the cut of (#350)', async (t) => { + const held = new Set(CUT_BLOCKS.keys()) + const { seeder, entry } = groupFixture(held, CUT_BLOCKS) + for (const seq of [1, 2, 3, 4, 5, 6, 7, 8, 10]) await seeder._pageArrived(GROUP_DIARY, entry, seq) + // Refused at their joins (a boxless writer of a private group, a core + // past its quota): only a lower cut on the writer lifts that. + entry.groupRefused.set(CUT_WRITERS[0], 1) + entry.groupRefused.set(CUT_WRITERS[1], 2) + + await seeder._pageArrived(GROUP_DIARY, entry, 9) + t.absent(entry.groupRefused.has(CUT_WRITERS[0]), 'the writer the late cut released is tried again') + t.is(entry.groupRefused.get(CUT_WRITERS[1]), 2, 'a writer it does not cut stays refused') +}) + // A member joined a writer (seq 1), left (seq 2, the join off) and deleted // the join (seq 3). The scan passed the join as a hole: when it arrives the // walk counts the writer, which readers then check by that on join. From 25fba70de3b18219f7f0a70d4078a60dd2891d68 Mon Sep 17 00:00:00 2001 From: rodrigopavezi Date: Tue, 17 Nov 2026 04:23:52 -0300 Subject: [PATCH 25/25] docs: describe the merge of main and the retired writer cut --- docs/protocol.md | 2 +- docs/verification/feat-writer-slot-sparse.md | 68 +++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/docs/protocol.md b/docs/protocol.md index 10fd97ca..b1bdb443 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -2185,7 +2185,7 @@ Added to section 7 "Seeder index route", under the same caps, Host check, 405 an - **Round 13, core node.** (1) A new writer key of a group is refused (`group-skipped` reason `quota`) while the group's foreign writer cores in the admission table plus those stored on disk, including writers kept after a quota stop, reach `MAX_GROUP_WRITERS` (`runSync` option `maxGroupWriters`; the group root is no writer slot and is never refused by this cap); a stored writer keeps its place, and a refused key is gated again once a place frees. (2) Block-0 divergences of group writers become one hard-error line `bloco 0 divergente em escritores de grupo: ()` (the result is still not ok), and a run keeps at most 32 hard-error strings. - **Membership and purges.** A fetched historical join block proves binding only. Current membership comes from the writer fold above. A stored writer core is purged only for an ACCEPTED writer (one that passed the gate): when D is blocked or its issuer is revoked, on a fork, or when its local block 0 differs from the gated one. A bad hint from a seeder never purges an unaccepted core. A blocked group root fails and purges every foreign core of that group with reason `blocked`, in the core node and in the browser alike; the member's own cores are never touched, and what is stored can still be read locally until it is purged. - **Binding diary release (core node).** A group writer's binding diary is downloaded and kept in the main store only while something needs it: a follow holds it, a writer in the admission table names it, or a stored foreign writer or root names it (the census is rebuilt from disk at start). When a writer is purged, retires or is evicted and nothing else needs its diary, the node stops downloading the diary and purges it from the main store. A writer stopped for quota keeps its diary (and its blocks). At run start a binding diary recorded in `group-certs.json` that no stored core names and that no follow holds is purged. At core node start, a member of any group also purges an unfollowed, unlisted diary unless a stored group writer or root names it or `group-certs.json` records it; that cleanup always rebuilds the census first, also when `groups.json` and `group-certs.json` are empty, and never purges a group core this node can write. A delete of a `group-join` block has no effect (section 2 "Deletion"): a held one keeps its writer bound and downloaded. One the node does not hold is fetched once (the binding diary is group-bound), and the gate waits a bounded time for it; only when the block is still missing after that wait does the core node drop the writer from the run (`group-skipped` reason `withdrawn`), without purging its blocks, and the group goal stops waiting for it. At browser node start, a stored replica that a stored group core binds (or that the run holds as a binding diary) stays indexed and is never purged by the unfollowed-replica cleanup. -- **Completion goals** (`--once` and the browser's once runs). Each group goal runs once per run and is met when (a) every writer page of every joined group was fetched (not `incomplete`) or the channel timed out, (b) every admitted foreign writer, and the diary that binds it up to its join, reached the length its seeder advertised, exactly like followed replicas (on the core node a core admitted through quarantine counts only after the block 0 its main store received was compared with the gated one), and (c) own writers were pushed to the length the seeder advertises for them (browser: confirmed by `group-pushed { key, length }`); all bounded by the run's deadline. A writer a seeder keeps sparse ("Sparse copies") never reaches its signed length contiguously, so (b) also holds for a foreign writer held up to the advertised length and past it with a hole once every connected peer that synced has answered for the blocks below the signed length (up to `MAX_LOG_BLOCKS`) and every block one of them advertises there is held (`advertisedHeld` in `sparse-held.js`, checked again every `SPARSE_CHECK_MS` = 100 ms while the goal waits; in the browser, which may run that goal for as long as a tab is open, the wait doubles up to 5 s while the copy's held range stays the same, and a download or a peer coming or going brings it back to 100 ms). Past a peer's contiguous length its bitfield arrives only as the answer to a download range's want: one want per bitfield segment (2097152 blocks), answered from the segment's first block up to the last page the peer holds anything in. So a peer counts as having answered once the first page of each segment of that range arrived; a later page it holds nothing in never comes. (c) for an own writer the own diary cut asks the seeder to hold it up to `max(cut, 1)` (`diaryWriterCut` over the diary's slot walk: 0 once released), never past the length the device holds, and past the cut each of its own `group-leave` and `group-key` blocks, as the seeder's replication peer advertises it or as served to that peer on this connection (a peer never announces a block it gets to one that already holds it). The device asks that peer for its bitfield there (`askHeld` in `sparse-held.js`), since a full copy sends no wants of its own; the core node reads the cuts when the run opens its writers, the browser at each scan of its diary. The account service's group fold for a page (`apps/cli/lib/account/pages-groups.js`, section 12) counts a listed writer whole on the same rule, so a group with such a writer is no longer partial (`indisponivel`): its fetch ends, before `FETCH_TIMEOUT_MS`, once every block the seeder advertises below the signed length is held (polled every 50 ms). Roots and diaries keep the contiguous rule. Against a seeder without `varanda/groups/1` (groups unsupported) the browser's own group pushes are not once-run goals, so a once run finishes. The goal of a binding diary is met at its present length (section 2 "Clearing deleted bytes"), so a deleted post no longer keeps a once run open, and join blocks after a cleared hole are read. Group writer and root cores keep their own tracking, since no delete records exist for them. Goals G1 to G3 of section 6 are unchanged. The browser's write gate for a group write confirms the own writer's length, not only the diary. For each community the goals are C's root, every page of its CWs, and the binding diaries, as for groups. +- **Completion goals** (`--once` and the browser's once runs). Each group goal runs once per run and is met when (a) every writer page of every joined group was fetched (not `incomplete`) or the channel timed out, (b) every admitted foreign writer, and the diary that binds it up to its join, reached the length its seeder advertised, exactly like followed replicas (on the core node a core admitted through quarantine counts only after the block 0 its main store received was compared with the gated one), and (c) own writers were pushed to the length the seeder advertises for them (browser: confirmed by `group-pushed { key, length }`); all bounded by the run's deadline. A writer a seeder keeps sparse ("Sparse copies") never reaches its signed length contiguously, so (b) also holds for a foreign writer held up to the advertised length and past it with a hole once every connected peer that synced has answered for the blocks below the signed length (up to `MAX_LOG_BLOCKS`) and every block one of them advertises there is held (`advertisedHeld` in `sparse-held.js`, checked again every `SPARSE_CHECK_MS` = 100 ms while the goal waits; in the browser, which may run that goal for as long as a tab is open, the wait doubles up to 5 s while the copy's held range stays the same, and a download or a peer coming or going brings it back to 100 ms). Past a peer's contiguous length its bitfield arrives only as the answer to a download range's want: one want per bitfield segment (2097152 blocks), answered from the segment's first block up to the last page the peer holds anything in. So a peer counts as having answered once the first page of each segment of that range arrived; a later page it holds nothing in never comes. (c) for an own writer the own diary cut asks the seeder to hold it up to `max(cut, 1)` (`diaryWriterCut` over the diary's slot walk: 0 once released), never past the length the device holds, and past the cut each of its own `group-leave` and `group-key` blocks, as the seeder's replication peer advertises it or as served to that peer on this connection (a peer never announces a block it gets to one that already holds it). The device asks that peer for its bitfield there (`askHeld` in `sparse-held.js`), since a full copy sends no wants of its own; the core node reads the cuts when the run opens its writers, the browser at each scan of its diary. An own writer the device retired by a join (`retiredWriterCut`, "Writers named by a node") is cut at the length it holds, so nothing past its cut waits for the seeder, unless the cut went at a lower mark already on it: its leaves and key blocks past that mark are confirmed the same way. The account service's group fold for a page (`apps/cli/lib/account/pages-groups.js`, section 12) counts a listed writer whole on the same rule, so a group with such a writer is no longer partial (`indisponivel`): its fetch ends, before `FETCH_TIMEOUT_MS`, once every block the seeder advertises below the signed length is held (polled every 50 ms). Roots and diaries keep the contiguous rule. Against a seeder without `varanda/groups/1` (groups unsupported) the browser's own group pushes are not once-run goals, so a once run finishes. The goal of a binding diary is met at its present length (section 2 "Clearing deleted bytes"), so a deleted post no longer keeps a once run open, and join blocks after a cleared hole are read. Group writer and root cores keep their own tracking, since no delete records exist for them. Goals G1 to G3 of section 6 are unchanged. The browser's write gate for a group write confirms the own writer's length, not only the diary. For each community the goals are C's root, every page of its CWs, and the binding diaries, as for groups. - **Parent paging.** For every local group root held the node reads its header's `community`, and for every local community its key. For each community C it pages C on the groups channel (the root and its CWs), gates each CW like a group writer (a valid header with `group` = C and `member: null`, then the binding diary's block 0 and join block), stores the CWs and their binding diaries, and pages only the roots of C's catalogue (the `/index/community/` rows are a hint, the fold verifies) so that `community.info` can name them. Each root is judged by its own block 0 under a role fixed for the run: a key joined as a group must be a `group` root, a key reached as a parent must be a `community` root, and a `v: 1` header is `header-invalid`; the `group-skipped` reason `not-community` (a parent that is not a community root; the node's own verdict, the seeder is not charged) is kept. A run holds at most groups x 512 plus communities x 512 cores. A node folding a group passes `buildGroup` the parent's input (C's root, its CWs and their joins). The core sync emits `community-parent { key, group }` (`key` the community root C, `group` the group G) the first time a held group root names a community this node does not already hold, and `group-parent { key: C, group: G }` once per local group whose held root names C, held or not (the session uses it to fan `groups.changed` out from C's growth to its groups; the CLI prints neither). - **Diary-bound roots (browser).** Besides kv `groupWriters` and kv `groupReplicas`, the browser sync pages every root its own diary binds now as a replica: the newest `group-join` per writer still `on: true`, among the writers the slot walk counts live or frozen per group ("Limits"), the first 64 groups by live join seq (`diaryGroupKeys`, the seeder's live-join rule). The scan keeps the diary's `group-cut` records with its joins, so a writer that took a slot a cut released binds its root, and the same walk picks the newly bound writers it chases. It reads them at run start from the stored diary and again whenever the own diary is caught up with the seeder, so a second device of the same passkey (same diary, fresh storage) rebuilds its communities and groups from the diary. A writer the diary newly binds to a root already adopted is paged at once (that group alone, ahead of the pass in progress or in a pass of its own), and again after waits of 2, 5 and 10 s (each counted from the previous page, so at about 2, 7 and 17 s) while the seeder does not list it yet; the writers of the run's first scan are not chased, the regular pass asks for them. A diary block the read could not get (not downloaded yet, or a read that failed) is kept, read again on every later scan, when it downloads and, after a failed read, on a short timer, and never holds back the joins after it. A once run does not end while that read of the own length is pending: the roots it adopts are paged before the run's goals are met. Such a root's writers are gated like any replica's, a catalogue child the diary joined is not paged root only (one already paged root only in the run is paged again in full), and a community among them is paged in full with its catalogue. Its own header decides its kind: another root's header naming it as its community never makes it a parent key, so it is never refused as `not-community`. A root no join binds now (every device left) is not adopted, a blocked key never is, and a key with no root at the seeder is dropped by the probe rule and not asked for again in that run. Nothing is appended, no writer is created and nothing is listed in kv for an adopted key until its block 0 is stored (the gate then lists it in kv `groupReplicas`). The core node is unchanged: its diary is per device (section 1), so another device never shares it. - **Automatic device enrolment (browser).** After a sync, a device of the same passkey enrols by itself in each diary-bound root it holds and has no kv `groupWriters` entry for, while the fold says the diary is a `member` there and fewer than `MAX_GROUP_WRITERS_PER_DIARY` - 2 (6) writers of the diary are live in its slot walk there (the root slot included): the last two live slots are spent only by an explicit join. A slot comes back only through the diary's own cut (a disconnected device, section 1, or the cut a join writes on a writer it replaces, "Writers named by a node"), while the walk has room to freeze or release the writer; an older seeder never releases one. It writes what `community.join` or `group.join` writes (a new writer naming the diary's latest leave, and the diary's public `group-join`), communities first, then a group only when this device's community writer is admitted in C (an open community or its creator; in an approval community the diary's admission covers the new community writer, "Admits list their writers", and its groups follow on a later run, once the fold shows it admitted). Geral is planned like any group of C with its own checks: a diary that left Geral is not put back, and Geral's last two slots stay reserved too. A kv writer the diary never bound (an interrupted join) is bound again, reusing that writer only when this device derives its key from the entry's nonce; otherwise the root is skipped (`foreign`), and an explicit join writes a new writer instead. Only a writer this device derived is ever reused by a join. A root that is left, banned, requested, blocked, unheld, an orphan, not joined by the own diary (an opened link) or not foldable yet (no pending join) is skipped, and no admit or grant is written automatically (the fold's coverage of a new device by the diary's admission writes nothing). Rules in [rpc.md](rpc.md) 5.63. The core node is unchanged (its diary is per device, section 1). diff --git a/docs/verification/feat-writer-slot-sparse.md b/docs/verification/feat-writer-slot-sparse.md index 2f18cb34..83cb21a8 100644 --- a/docs/verification/feat-writer-slot-sparse.md +++ b/docs/verification/feat-writer-slot-sparse.md @@ -51,6 +51,32 @@ Quarta rodada (revisão de `d487e5e7..9b2137f8`, sem P1 nem P2): - Releitura da faixa incerta quando um par entra (P3-a, `_rereadIfOffered` em `seeder.js`, `peerOffers` em `sparse-held.js`): só relê na hora quando o par que entrou, depois de sincronizar e responder ao pedido do bitfield dessa faixa (`askHeld`), anuncia ali um bloco que o semeador não tem. Conferido a cada 1 s desde a entrada, até 10 vezes. Qualquer outro par (um leitor que conecta, um par que reconecta em laço) deixa o recuo como está. Antes, cada `peer-add` voltava o recuo a 1 s e relia a faixa inteira, então o recuo de 5 minutos nunca valia para um escritor cujo aparelho sumiu. - `peerHolds` ganhou um teste de unidade com blocos servidos (a mutação R4 da terceira rodada passava). +Quinta rodada: merge da `main` em `4e901c15` (#717, #721, #722), commit `955ae568`, com a mensagem padrão do git. + +- Conflitos, resolvidos mantendo os dois lados: + - `seeder.js`: a varredura dobra o `group-cut` pelo `foldGroupRecord` da #717, dentro de `foldGroupCut`, que guarda a regra deste PR: um corte menor tenta de novo a entrada recusada. + - `sync.js` do nó core: os imports das duas. + - `sync.js` do navegador: + - os imports, o campo `deletedHoles` da #717 e as exportações das duas; + - `fold.cutEntries` saiu: a #722 já guarda os cortes em `fold.joinEntries` (`WALK_TYPES`), e `ownCutEntries` passa a ser essas entradas. + - `member-ops.js`: um comentário. + - `seeder-groups-cuts.test.js`: os imports e o teste novo da #721 antes do de liberação. + - `protocol.md`: o texto da `main` com as frases deste PR. O escritor sem caixa substituído é recusado pela sincronização, e o semeador o guarda como toco. `buildCommunity` recebe `signed?`. +- Além de escolher lados: + - `_pageArrived` (#717): um `group-cut` apagado, buscado uma vez depois da varredura, também passa por `foldGroupCut`. Um corte menor que chega assim levanta a recusa da entrada como na varredura. Há teste novo em `seeder-delete.test.js`, cujo fixture ganhou `groupRefused`. + - Teste de unidade da #721 "a stored binding the scan no longer lists gets its on join too": montava a entrada do diário sem `core` nem `scanned`. O `_reconcileGroups` deste PR chama `_groupCutScan`, que lê os dois, e o arquivo parava com `TypeError` no commit do merge. O fixture ganhou um diário lido até o fim e uma entrada servida. + - Teste "a cut that is gone after a restart fetches again what clearing removed": apagava o registro de corte para o corte sumir. Com a #717 um `group-cut` apagado continua valendo, então o semeador nunca limpava o bloco e o teste esgotava o tempo em "corte apagado no semeador". Virou dois testes: + - "a cut its diary deleted still applies at the seeder, after a restart too": o bloco e o corte ficam, também depois de reiniciar, e nada limpo é buscado de novo. + - "a persisted cut the diary does not give is dropped once the scan caught up, and what clearing removed is fetched again": o mesmo caminho de antes (`caught && cut !== now`), agora com um corte guardado no `admitted.json` que o diário não tem e W limpo depois dele com o semeador parado. +- Interações conferidas: + 1. **Teto de 32 escritores por diário e grupo (#722) e guarda esparsa.** O leitor conta núcleos guardados pelo seq da entrada provada (`othersHolding`, `MAX_GROUP_HELD_PER_DIARY`). O semeador conta ligações por diário e grupo, com o mesmo 8 + 8 + 16. Nenhum dos dois olha os blocos, então um escritor cortado guardado esparso ocupa uma vaga como o inteiro. Quem conta, nos dois, é a caminhada do diário. A guarda esparsa só muda quais blocos ficam, e a dobra (`bindCores`) dobra a cópia esparsa como a inteira. + 2. **Cortes de substituição automáticos (#722).** `retiredWriterCut` põe o corte um depois do último seq que o aparelho tem, isto é, no comprimento do escritor; o teste 7 de `community-member-leave-5.test.js` confere isso. + - A meta do dono vira `min(comprimento, max(corte, 1))`, que é o comprimento. No nó core `ownKeptMissing` nem é chamado, e no navegador `seederHoldsKept` devolve `true` (`cut >= length`). + - Só quando o corte vai na marca menor que já estava no escritor sobram saídas e chaves depois dela. Elas são confirmadas como as de qualquer corte, pelo caminho que Q1, Q2, R1 e R3 seguram. + - O escritor aposentado também deixa de ser o `writer` do grupo em `groups.json`, então nem entra na meta. + 3. **O seq da listagem (#721, `onSeq`) é um seq do diário.** Ele vem da caminhada que a varredura do diário monta. A guarda esparsa só vale para núcleos de escritor (raízes e diários seguem pelo contíguo), então não o muda. O teste novo da #721 passa com a guarda esparsa. + 4. **Busca única dos blocos apagados mantidos (#717).** Acontece nos diários (`entry.deleted`, `groupSeqs`, `fold.deletedHoles`). `keptPast`, `keptPastBytes` e a cota do que fica são dos núcleos de escritor, que não têm registros de apagar. O único encontro é um `group-cut` apagado que chega depois: ele baixa o corte pelo caminho da varredura (`_reconcileGroups`, `_groupCutScan`, `_applyGroupCut`, que tira de `keptPast` o que fica abaixo do novo `clearedFrom`), e agora também levanta a recusa. + ## Como verificar Cada arquivo sozinho, a partir da raiz, com `node `, e `npx standard` nos arquivos mudados: @@ -127,6 +153,37 @@ Quarta rodada, cada arquivo sozinho com `VARANDA_TEST_CONCURRENCY=1 node `. Primeiro no commit do merge (`955ae568`), depois de novo nos arquivos que os ajustes mexem: + +| Arquivo | Testes | Asserções | +|---|---|---| +| `packages/core/test/sparse-held.test.js` | 4/4 | 18/18 | +| `packages/core/test/sync-groups-sparse.test.js` | 2/2 | 20/20 | +| `packages/core/test/seeder-groups-cuts.test.js` (no merge: `TypeError` no teste da #721; depois do ajuste do fixture e dos dois testes de corte apagado) | 20/20 | 361/361 | +| `packages/core/test/sync-groups-3.test.js` (rodado de novo depois do corte tardio) | 26/26 | 143/143 | +| `packages/core/test/group-member.test.js` | 15/15 | 158/158 | +| `packages/core/test/group-member-6.test.js` | 10/10 | 72/72 | +| `packages/core/test/sync-private.test.js` | 4/4 | 16/16 | +| `packages/core/test/seeder-delete.test.js` (35/35, 288/288 no merge; com o teste do corte tardio) | 36/36 | 290/290 | +| `packages/core/test/sync-groups-7.test.js` | 1/1 | 12/12 | +| `packages/core/test/writer-slot-member.test.js` | 9/9 | 61/61 | +| `packages/core/test/sync-groups-2.test.js` | 27/27 | 96/96 | +| `packages/core/test/group-rekey.test.js` | 16/16 | 118/118 | +| `packages/core/test/community-member-leave-5.test.js` | 10/10 | 86/86 | +| `packages/core/test/deletion.test.js` (rodado de novo depois do corte tardio) | 65/65 | 963/963 | +| `packages/core/test/community-fold.test.js` | 121/121 | 1111/1111 | +| `packages/web-member/test/sync-groups-sparse.test.js` | 3/3 | 27/27 | +| `packages/web-member/test/sync-groups-2.test.js` | 29/29 | 139/139 | +| `packages/web-member/test/private-ops.test.js` | 24/24 | 231/231 | +| `packages/web-member/test/group-memo.test.js` | 12/12 | 65/65 | +| `packages/web-member/test/sync-groups-3.test.js` | 1/1 | 14/14 | +| `packages/web-member/test/group-ops.test.js` | 64/64 | 487/487 | +| `packages/web-member/test/community-ops.test.js` | 77/77 | 564/564 | +| `apps/cli/test/pages-groups-sparse.test.js` | 1/1 | 8/8 | +| `apps/cli/test/pages-groups.test.js` | 4/4 | 157/157 | + +`npm run lint` passa. + Não rodados aqui, por regra do brief: a suíte inteira e a demo (a sessão que abriu esta roda a CI de entrega). ## Prova de regressão @@ -202,6 +259,15 @@ Quarta rodada, uma mutação por vez: | T4 um par que oferece um bloco não relê nada | o mesmo | tempo esgotado "W relido inteiro" (teste 15: com o recuo em 5 minutos, só o anúncio de B relê) | | T3 `peerHolds` sem os blocos servidos (R4) | `packages/core/test/sparse-held.test.js` | 3/4, 17/18, falha "a block served to it counts" | +Quinta rodada, depois do merge, uma mutação por vez: + +| Mutação | Arquivo | Resultado | +|---|---|---| +| U1 `_pageArrived` dobra o corte apagado tardio só com `foldGroupRecord` (o código da #717) | `packages/core/test/seeder-delete.test.js` | 35/36, 289/290, falha "the writer the late cut released is tried again" (teste 32) | +| V1 sem `caught && cut !== now` (a M4 de antes) | `packages/core/test/seeder-groups-cuts.test.js` | tempo esgotado "W buscado de novo" (teste 15) | + +A M4 rodou sobre o teste que apagava o registro de corte, que a #717 tornou impossível (acima); a V1 é a mesma mutação sobre o teste que o substitui. No commit do merge, antes do ajuste do fixture, `seeder-groups-cuts.test.js` parava com `TypeError` em `scanCaughtUp` no teste da #721: é o encontro do `_groupCutScan` deste PR com o `_reconcileGroups` da #721, não uma mutação. + ## Decisões tomadas aqui - A regra do buraco mora na dobra (`bindCores`), não só nos leitores: assim o nó core, o nó do navegador e o serviço de contas dobram igual mesmo com cópias diferentes do mesmo escritor, e um bloco depois de um buraco abaixo do corte nunca conta. @@ -224,7 +290,7 @@ Quarta rodada, uma mutação por vez: - (a) e (b) feitos na segunda rodada (acima). - (c), (d) e (e): decisões mantidas (acima). Riscos que ficam: (c) uma dobra de catálogo vê menos de um CW cortado do que a dobra completa (só o prefixo), nunca mais; (d) um escritor esparso perto da cota de um leitor conta pelo comprimento assinado, não pelo que o leitor guarda; (e) a banda do semeador para um escritor cortado longo continua a do escritor inteiro até `MAX_LOG_BLOCKS`. - (f) Não feito: parado para decisão (pedido da rodada). As sincronizações (core e navegador) ainda recusam com `header-invalid` o cabeçalho sem caixa num grupo privado antes de guardá-lo, então um leitor que nunca o teve fica com a dobra `incomplete`, embora o semeador mantenha o toco. Fazer exige que o veredito do portão dependa da caminhada do diário que liga o escritor (o registro de corte 0, não só o bloco de entrada que o I1 admite), levantar a recusa da rodada (`refused`) quando essa caminhada muda, guardar só `[0, 1)`, pular esse escritor em `recheckGroupBoxes`, na revalidação da partida e na meta de conclusão, e decidir o que acontece quando a liberação some (registro de corte apagado). Isso muda as regras de admissão (I1 a I4) nas duas sincronizações. -- (g) Encontrado na segunda rodada, anterior a este PR (vem da #713): o semeador lista um escritor que o diário desligou com `group.leave` com o seq desse `group-join` `on: false`, e o portão do leitor exige `on: true` naquele seq, então recusa como `unbound` (no navegador, visto no teste: `group-skipped` com `reason: 'unbound'`; no nó core pela leitura do código, `joinVerdict`), cobra como dica ruim do semeador e nunca guarda o escritor. Com o diário guardado, a caminhada espera o escritor e a dobra pode ficar `incomplete`. Não mexido aqui. +- (g) Encontrado na segunda rodada, anterior a este PR (vem da #713): o semeador lista um escritor que o diário desligou com `group.leave` com o seq desse `group-join` `on: false`, e o portão do leitor exige `on: true` naquele seq, então recusa como `unbound` (no navegador, visto no teste: `group-skipped` com `reason: 'unbound'`; no nó core pela leitura do código, `joinVerdict`), cobra como dica ruim do semeador e nunca guarda o escritor. Com o diário guardado, a caminhada espera o escritor e a dobra pode ficar `incomplete`. Não mexido aqui. Quinta rodada: a #721, que entrou no merge, lista o escritor que saiu pelo seq do `group-join` `on: true` mais novo (`onSeq`). Conferido só pela leitura do código e pelo teste de unidade da #721, que passa aqui; o teste do navegador com `group.leave` não foi refeito. - O "respondeu" de `advertisedHeld` é uma aproximação: a primeira página de um segmento criada por outra mensagem do par (o anúncio de um bloco só, um `range` de comprimento 1, por exemplo) antes da resposta ao pedido faz o par contar como respondido cedo. A faixa contígua do par não cria página, então isso só acontece com um bloco anunciado avulso nos primeiros 32768 blocos do segmento. O efeito é um `--once` terminar antes de um bloco anunciado chegar: a dobra trata a cópia como o prefixo (o que está depois do buraco e não chegou não conta) até a próxima sincronização, sem erro. - Um semeador que só tem pares esparsos de um escritor cortado guarda o que eles têm depois do corte; uma saída ou um bloco de chave que nenhum par conectado tem só chega quando um par mais completo aparece (a faixa incerta é relida na hora quando um par que entra anuncia ali um bloco que falta e, sem isso, a cada até 5 minutos). Um par que só tem esse bloco depois da faixa contígua dele conta pela resposta ao pedido do bitfield; se ela não chega em 10 s, vale o recuo. - No nó core, os cortes do próprio diário são lidos uma vez, quando a sincronização abre os escritores: um corte escrito durante a mesma execução só vale na próxima.