diff --git a/apps/cli/lib/account/pages-groups.js b/apps/cli/lib/account/pages-groups.js index 82b99d51..74e43dc6 100644 --- a/apps/cli/lib/account/pages-groups.js +++ b/apps/cli/lib/account/pages-groups.js @@ -45,12 +45,13 @@ const hcrypto = require('hypercore-crypto') const Hypercore = require('hypercore') const Corestore = require('corestore') const { WebSocket, createWebSocketStream } = require('ws') -const { writeJsonAtomic, appendRecord, readAll, readBlocks, openReplica, purgeReplica } = require('@varanda/core/lib/store') +const { writeJsonAtomic, appendRecord, readAll, readBlocks, readHeldBlocks, openReplica, purgeReplica } = require('@varanda/core/lib/store') const { refuseSymlinkDir, certForKey, seederCertForKey, isPlainObject } = require('@varanda/core/lib/cert') const { attachAdmission } = require('@varanda/core/lib/admission') const { attachGroups, GROUPS_LIMIT_MAX } = require('@varanda/core/lib/groups-net') const { generateNoiseKeyPair } = require('@varanda/core/lib/net') const { groupJoinRecord, latestProfile, validateRecord, MAX_LOG_BLOCKS } = require('@varanda/core/lib/records') +const { advertisedHeld } = require('@varanda/core/lib/sparse-held') const { latestUsername } = require('@varanda/core/lib/username') const groups = require('@varanda/core/lib/group') const { keyTag, keyCommit, wipe } = require('@varanda/core/lib/group-crypto') @@ -67,6 +68,9 @@ const NOISE_FILE = 'noise-key.json' const MAX_REPLICAS = 1024 const MAX_WRITER_PAGES = Math.ceil(groups.MAX_GROUP_WRITERS / GROUPS_LIMIT_MAX) const FETCH_TIMEOUT_MS = 10000 +// A listed writer held past the seeder's prefix is checked this often for +// the blocks the seeder advertises (core sparse-held.js). +const SPARSE_POLL_MS = 50 const BACKOFF_MIN_MS = 1000 const BACKOFF_MAX_MS = 60000 const HANDSHAKE_TIMEOUT_MS = 10000 @@ -604,16 +608,34 @@ function createPagesGroups ({ dir, writer, pagesStore, keys, association, blockl // The core at the length the seeder offers, held (release it), or null // when nothing arrived in time. `complete` is false while a block up to - // min(length, MAX_LOG_BLOCKS) is missing. - async function fetch (key, held) { + // min(length, MAX_LOG_BLOCKS) is missing, or for a `sparse` core (a listed + // writer, which a seeder keeps sparse once its diary cut it, #350) while + // a block the seeder advertises below that is missing. + async function fetch (key, held, sparse = false) { const core = await hold(key) held.push(key) await timeout(core.update({ wait: true }).catch(noop), fetchTimeoutMs) if (core.length === 0) return null const end = Math.min(core.length, MAX_LOG_BLOCKS) - await timeout(core.download({ start: 0, end }).done().catch(noop), fetchTimeoutMs) + const range = core.download({ start: 0, end }).done().catch(noop) + if (!sparse) { + await timeout(range, fetchTimeoutMs) + if (core.contiguousLength === 0) return null + return { core, complete: core.contiguousLength >= end } + } + // The range never completes past a seeder's hole: it ends once every + // block the seeder advertises is here. + const deadline = Date.now() + fetchTimeoutMs + let done = false + range.then(() => { done = true }) + let complete = false + for (;;) { + complete = core.contiguousLength >= end || (core.contiguousLength > 0 && await advertisedHeld(core, end)) + if (complete || done || closed || Date.now() >= deadline) break + await timeout(range, Math.min(SPARSE_POLL_MS, Math.max(0, deadline - Date.now()))) + } if (core.contiguousLength === 0) return null - return { core, complete: core.contiguousLength >= end } + return { core, complete } } function localWriter (key) { @@ -699,7 +721,7 @@ function createPagesGroups ({ dir, writer, pagesStore, keys, association, blockl // replica of the same key would never get a channel, so read it here. let got = localWriter(it.key) try { - if (got === null) got = await fetch(it.key, held) + if (got === null) got = await fetch(it.key, held, true) } catch { incomplete = true continue @@ -711,7 +733,9 @@ function createPagesGroups ({ dir, writer, pagesStore, keys, association, blockl if (!got.complete) incomplete = true const core = got.core if (core.fork !== 0) continue - const entries = await readBlocks(core) + // A seeder keeps a cut writer sparse (#350): past its first hole, its + // leaves and key blocks. + const entries = await readHeldBlocks(core) const h = entries[0] && entries[0].record if (groups.validateWriterHeader(h) !== null || h.group !== key) continue logs.push({ key: it.key, diary: h.diary, entries, length: core.contiguousLength, signed: core.length }) diff --git a/apps/cli/test/pages-groups-sparse.test.js b/apps/cli/test/pages-groups-sparse.test.js new file mode 100644 index 00000000..3c7e92ee --- /dev/null +++ b/apps/cli/test/pages-groups-sparse.test.js @@ -0,0 +1,138 @@ +'use strict' + +// apps/cli/lib/account/pages-groups.js with a writer a seeder keeps sparse +// (#350, docs/protocol.md "Sparse copies"): its blocks below the diary cut, +// then only its leaves and key blocks. The page's fold counts such a copy +// whole once every block the seeder advertises is held. Its own file: the +// pages-groups.test.js runs already take most of the runner's per-file time. + +const test = require('brittle') +const b4a = require('b4a') +const crypto = require('hypercore-crypto') +const { testnet, tmpDir, until } = require('./helpers/net') +const { createIssuerCert, createIssuedCert, createUsernameRecord, generateAssociationKeyPair } = require('@varanda/core/lib/cert') +const { signRoster, issuerCountersign } = require('@varanda/core/lib/page') +const member = require('@varanda/core/lib/member') +const coreStore = require('@varanda/core/lib/store') +const { groupPost, groupLeave } = require('@varanda/core/lib/group') +const { groupCutRecord } = require('@varanda/core/lib/records') +const { person, exchange, startSeeder, syncOnce, ownWriter, runUntil } = require('../../../packages/core/test/helpers/sync-groups') +const { offlineAppend, writerEntries } = require('../../../packages/core/test/helpers/group-member') +const { startSeedWs } = require('../lib/seed-ws') +const { openPageKeys } = require('../lib/account/page-keys') +const { createPagesWriter } = require('../lib/account/pages-writer') +const { openPagesStore } = require('../lib/account/pages-store') +const { createPagesGroups } = require('../lib/account/pages-groups') + +const PAGE_KEYS = 'cd'.repeat(32) + +function hex (buf) { + return b4a.toString(buf, 'hex') +} + +let nonceCount = 0 +function nonce () { + nonceCount++ + return (String(nonceCount) + 'g'.repeat(43)).slice(0, 43) +} + +function newPage (ctx, keys, username) { + const { key, row } = keys.pending(nonce()) + keys.put(key, row) + const cert = createIssuedCert({ member: key }, ctx.issuerKp, ctx.issuerCert) + const usernameRecord = createUsernameRecord({ member: key, username }, ctx.issuerKp, ctx.issuerCert) + const owner = crypto.keyPair() + const fields = { page: key, gen: 1, owner: hex(owner.publicKey), admins: [], prev: null, at: new Date().toISOString(), nonce: nonce(), association: ctx.association } + const signature = signRoster(fields, owner) + const rosterRecord = issuerCountersign({ type: 'page-roster', ...fields, signature }, ctx.issuerKp, ctx.issuerCert) + return { key, owner: hex(owner.publicKey), args: { cert, usernameRecord, rosterRecord } } +} + +// Retries while the fold is still partial ('indisponivel'). +function settled (fn, name, timeoutMs) { + return until(async () => { + try { + return await fn() + } catch (err) { + if (err.code === 'indisponivel') return null + throw err + } + }, { name, timeoutMs, intervalMs: 500 }) +} + +async function blocksHeld (core, from, to) { + const out = [] + for (let i = from; i < to; i++) if (await core.has(i)) out.push(i) + return out +} + +test('pages-groups: a group with a writer the seeder keeps sparse is no longer indisponivel (#350)', async (t) => { + t.timeout(300000) + const { bootstrap } = await testnet(t) + const kp = generateAssociationKeyPair() + const association = hex(kp.publicKey) + const issuerKp = crypto.keyPair() + const ctx = { association, issuerKp, issuerCert: createIssuerCert({ issuer: hex(issuerKp.publicKey) }, kp) } + const seeder = await startSeeder(t, bootstrap, kp) + const server = await startSeedWs({ port: 0, seeder }) + t.teardown(() => server.close(), { force: true }) + const seederWs = 'http://127.0.0.1:' + server.port + + const a = await person(t, kp, 'Ana') + const b = await person(t, kp, 'Bia') + const { community: C } = await member.communityCreate(a, { name: 'Bairro', join: 'open' }) + const { group: G } = await member.groupCreate(a, { name: 'Praça', join: 'open', community: C }) + await member.groupPost(a, { group: G, text: 'oi' }) + await exchange(a, b) + await member.communityJoin(b, { community: C }) + await member.groupJoin(b, { group: G }) + await member.groupPost(b, { group: G, text: 'antes' }) + // W, B's writer in G, is frozen at its length: a post and a leave past + // the cut. + const w = ownWriter(b, G) + const cut = (await writerEntries(b, G)).length + await offlineAppend(b, G, groupPost({ text: 'depois' })) + const leave = await offlineAppend(b, G, groupLeave({ seen: null, cuts: [{ writer: w, seq: cut + 1 }] })) + const s = await coreStore.openStore(b.dataDir) + try { + await coreStore.appendRecord(await coreStore.openDiary(s), groupCutRecord({ group: G, writer: w, cut })) + } finally { + await s.close() + } + t.ok((await syncOnce(bootstrap, a)).result.ok, 'A pushes its diary and roots') + await runUntil(b, bootstrap, async () => { + const e = seeder._groupServed.get(w) + const st = seeder._groups.get(w) + if (!e || !st || e.cutting || st.cut !== cut || st.clearedTo !== leave.seq + 1) return false + return (await blocksHeld(e.core, 0, leave.seq + 1)).length === cut + 1 + }) + t.alike(await blocksHeld(seeder._groupServed.get(w).core, 0, leave.seq + 1), [...Array(cut).keys(), leave.seq], 'the seeder keeps W below its cut and its leave') + + const dir = tmpDir(t, 'varanda-account-') + const keys = openPageKeys({ dir, env: { VARANDA_PAGE_KEYS: PAGE_KEYS } }) + const pagesStore = openPagesStore(dir) + const writer = createPagesWriter({ dir, keys, association, seederWs }) + t.teardown(() => writer.close(), { force: true }) + await writer.ready() + const pg = createPagesGroups({ dir, writer, pagesStore, keys, association, seederWs }) + t.teardown(() => pg.close(), { force: true }) + await pg.ready() + await until(() => writer.connected() && pg.connected(), { name: 'links up', timeoutMs: 20000 }) + const page = newPage(ctx, keys, 'padaria') + await writer.createPage(page.args) + await pagesStore.create({ page: page.key, owner: page.owner }) + + t.is((await settled(() => pg.communityJoin(page.key, C), 'community join', 60000)).state, 'member', 'the page joins the community') + // Each try fetches W again; before #350's goal it waited out the fetch + // timeout and answered indisponivel every time. + t.is((await settled(() => pg.groupJoin(page.key, G), 'group join', 60000)).state, 'member', 'the page joins the group with W sparse') + // The seeder indexes the page's new writer first: its listing is partial + // meanwhile. + const posted = await settled(() => pg.groupPost(page.key, G, { text: 'pão quente' }), 'group post', 60000) + t.ok(Number.isSafeInteger(posted.seq), 'and posts') + const read = await pg.groupRead(page.key, G) + t.is(read.incomplete, false, 'its fold is whole') + const texts = read.items.map((p) => p.text) + t.ok(texts.includes('antes'), 'the post below the cut counts') + t.absent(texts.includes('depois'), 'the post past the cut does not') +}) diff --git a/docs/changelog/feat-writer-slot-sparse.md b/docs/changelog/feat-writer-slot-sparse.md new file mode 100644 index 00000000..080090de --- /dev/null +++ b/docs/changelog/feat-writer-slot-sparse.md @@ -0,0 +1,3 @@ +### Alterado + +- Os semeadores passam a guardar só o que conta de um escritor cortado pelo diário (PR `feat/writer-slot-sparse`, parte da #350): os blocos abaixo do corte e, a partir dele, só as saídas do grupo e os blocos de chave. De um escritor liberado (corte 0) fica um toco: o bloco 0, as saídas e os blocos de chave. A cota conta o que fica guardado. O nó core, o nó do navegador e o serviço de contas leem um escritor além do primeiro buraco (só as saídas e os blocos de chave), então quem só fala com semeadores vê o grupo como quem tem as cópias inteiras. Um bloco de chave que cita um escritor de comunidade cortado vale pelo comprimento que o escritor assinou. O escritor sem caixa trocado num grupo privado, depois de liberado, também fica como toco no semeador. O `sync --once` (nó core e navegador) termina com um escritor guardado esparso quando todo bloco que os pares anunciam está guardado, e o dono dá o escritor cortado por enviado quando o semeador o guarda até o corte e já tem as saídas e os blocos de chave dele depois do corte. Um semeador que só recebe o escritor de pares que o guardam esparso também chega às saídas depois do corte. Entrar, postar ou selar pela Página num grupo com escritor esparso não responde mais `indisponivel`. Conferência em `docs/verification/feat-writer-slot-sparse.md`. diff --git a/docs/protocol.md b/docs/protocol.md index f665c896..84e4a180 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -1017,7 +1017,7 @@ A seeder `hello` additionally carries `"cert": { "v": 1, "seeder": "", "d - The admission channel's open handshake carries one compact-encoding `uint`, the sender's capability bits; with no bits set (caps 0) the open carries no bytes at all, so a peer without capabilities opens the channel byte for byte as before. Decoding is tolerant: an open with no bytes (every peer from before capabilities) or with bytes that do not parse reads as caps 0, and unknown bits are ignored, so either side may be older than the other. `attachAdmission` takes the option `caps` (a non-negative safe integer, else `TypeError`, default 0) and exposes the peer's bits as `Peer.remoteCaps`. - `CAP_NO_INVENTORY = 1`: sent by a member that wants no inventory from the seeder (the lean hello of "Bounded inventory" above). -- `CAP_WRITER_CUTS = 4`: sent by every seeder that keeps the writers a diary cut (section 10, "Seeder group admission"): per diary and group up to 8 live and 8 frozen writers chosen by the slot walk and up to 16 writers the diary released, each held whole under the whole-log quota. Members may use it to tell such a seeder from an older one, which counts at most 8 writers per diary and group by earliest join and never frees a slot. This version only announces it. +- `CAP_WRITER_CUTS = 4`: sent by every seeder that keeps the writers a diary cut (section 10, "Seeder group admission"): per diary and group up to 8 live and 8 frozen writers chosen by the slot walk and up to 16 writers the diary released, a frozen one kept below its cut plus its leaves and key blocks and a released one as a stub (block 0, leaves and key blocks), under the log quota on what is kept. Members may use it to tell such a seeder from an older one, which counts at most 8 writers per diary and group by earliest join and never frees a slot. This version only announces it. - `CAP_DISCOVERY = 2`: sent by a seeder that answers `'varanda/discovery/1'` and sends lean hellos. The seeder sends it unless it was started with `discovery: false` (constructor option, default true, no CLI flag: tests and rollback), in which case it behaves exactly like a seeder from before discovery: no bit, no lean hello, no discovery channel (a member's discovery open is rejected). - The capabilities change nothing else: the message shapes and limits of this section, `maxServed`, `maxExternal`, the frame limits and the hello of a peer that sends no bit are unchanged. @@ -1917,13 +1917,13 @@ A CW record (admit, ban, hide or invites) whose `group` names a key outside C's `MAX_GROUP_WRITERS_PER_DIARY` = 8: `buildGroup` counts per diary the bound writers its slot walk counts (below): at most 8 live ones, a bound root one of its creator's 8, plus at most `MAX_FROZEN_WRITERS_PER_DIARY` = 8 frozen ones. `MAX_GROUP_WRITERS` = 512: of the writers that remain, the fold keeps the root, then the other writers in this order of tiers: those of the creator's diary (root bound only), those of the diaries that are admins in the final root state (group admins, and the creator and admins of the group's community), all others, and last those of banned diaries (ranked by the final ban layer); inside a tier the live writers before the frozen ones, then by key. A rejoining writer whose named leave fell outside the cap is dropped as well, because the fold would otherwise never see that leave. `incomplete` is true when the 512 cap or that rejoin rule drops a writer; the per-diary cap alone does not set it. `MAX_GROUPS_PER_DIARY` = 64 is enforced per diary by the seeder, which keeps the 64 EARLIEST live joins by diary seq (the root counts toward the 512 writer cores of a group at the seeder). `MAX_GROUPS_PER_COMMUNITY` = 32 counts the `group-create` records of a community in its causal order ("Communities contain groups"). `MAX_GROUP_CUTS` = 16 caps the cuts of admits, bans and admins, and also of a community `group-admin` (over the member's CWs, of which a diary binds at most 8): there is no separate community cut cap. A community core set (the root C and its CWs) uses the same per-diary and total caps and the same tiers as a group. `MAX_GROUP_NAME` = 80. Blocks use `MAX_BLOCK_BYTES` = 16384; each writer core uses section 2's `MAX_LOG_BLOCKS` = 100000 and `MAX_LOG_BYTES` = 64 MiB. Text length is defined in the post shape above. -**Slot walk** (#350, `slotWalk` in `group.js`). Per diary d and root G (or community C), the counted writers come from d's own `group-join` and `group-cut` records for G in diary seq order and nothing else, so every reader and every seeder agrees, and a prefix of the diary never disagrees with the whole diary about the records it holds. A writer's state only moves forward: none, live, frozen, released. (1) An `on: true` join of a writer in state none makes it live while fewer than 8 are live (7 from d's first `on: true` join of the root key G itself: the root slot, which any diary joining writer G reserves from that join on); with no room the writer is over and never counts, even when a slot frees later. A repeated or `on: false` join changes no slot. (2) A mark on a live or frozen writer lowers its cut (the lowest wins, and it always applies to the writer's records, "Diary cuts on own writers"). Cut 0 releases the writer while d released fewer than `MAX_WRITER_RELEASES_PER_DIARY` = 16 writers in G; a cut >= 1 freezes a live writer while fewer than 8 are frozen; with no room the writer keeps its state and its slot, with the cut. (3) A mark on a writer in state none or released, every mark on the root, and a mark with no valid seq are ignored. The candidates are the live, frozen and released writers. A writer cut at 0 (released, or kept at cut 0 once the 16 releases are spent) gives its slot back when there is room, but its block 0, its leaves and its key blocks still count ("Diary cuts on own writers"), so it is awaited like a frozen writer, and a released one ranks after the frozen writers of its tier under the 512 cap. A released writer is awaited whatever its diary's newest join says, in a private group too, so a diary that turns off and cuts at 0 the writer holding its leave never gets a fold that looks complete without that leave. Once held, a writer cut at 0 whose header box does not match the group's kind (the boxless writer a node replaced, "Writers named by a node") is rejected, never awaited; seeders and sync refuse such a header with `header-invalid`, so a reader that never held it keeps the fold incomplete. A writer that is over is no candidate, is never awaited and never makes the fold incomplete. Releases bound the churn: a loop of join, cut 0, join stops after 16 releases and 8 writers held at cut 0, so a diary has at most 8 live, 8 frozen and 16 released writers in G. A rejoin naming a leave on a writer of the same diary that its walk voided (over) is rejected; one naming a leave of a released writer is valid, since that leave still counts. A hole in the history of a diary with an `on: true` join for G (`{ gap: true }` in its marks) may hide the mark that freed or froze a slot, so it sets `incomplete`. The fold returns `frozen` (the frozen writers, released ones included), `released` (the released writers it kept) and `slots` (diary -> `{ live, frozen, released }`, the counts of each diary with an `on: true` join for G, from which nodes derive the room left). `diaryGroupKeys` and the seeder ("Seeder group admission") use the same walk, so a second device of the account and the seeders open the same roots and writers. +**Slot walk** (#350, `slotWalk` in `group.js`). Per diary d and root G (or community C), the counted writers come from d's own `group-join` and `group-cut` records for G in diary seq order and nothing else, so every reader and every seeder agrees, and a prefix of the diary never disagrees with the whole diary about the records it holds. A writer's state only moves forward: none, live, frozen, released. (1) An `on: true` join of a writer in state none makes it live while fewer than 8 are live (7 from d's first `on: true` join of the root key G itself: the root slot, which any diary joining writer G reserves from that join on); with no room the writer is over and never counts, even when a slot frees later. A repeated or `on: false` join changes no slot. (2) A mark on a live or frozen writer lowers its cut (the lowest wins, and it always applies to the writer's records, "Diary cuts on own writers"). Cut 0 releases the writer while d released fewer than `MAX_WRITER_RELEASES_PER_DIARY` = 16 writers in G; a cut >= 1 freezes a live writer while fewer than 8 are frozen; with no room the writer keeps its state and its slot, with the cut. (3) A mark on a writer in state none or released, every mark on the root, and a mark with no valid seq are ignored. The candidates are the live, frozen and released writers. A writer cut at 0 (released, or kept at cut 0 once the 16 releases are spent) gives its slot back when there is room, but its block 0, its leaves and its key blocks still count ("Diary cuts on own writers"), so it is awaited like a frozen writer, and a released one ranks after the frozen writers of its tier under the 512 cap. A released writer is awaited whatever its diary's newest join says, in a private group too, so a diary that turns off and cuts at 0 the writer holding its leave never gets a fold that looks complete without that leave. Once held, a writer cut at 0 whose header box does not match the group's kind (the boxless writer a node replaced, "Writers named by a node") is rejected, never awaited; the member sync refuses such a header with `header-invalid` before storing it, so a reader that never held it keeps the fold incomplete; seeders refuse it while it is counted and keep it as a stub once released, so its block 0 stays available ("Seeder group admission"). A writer that is over is no candidate, is never awaited and never makes the fold incomplete. Releases bound the churn: a loop of join, cut 0, join stops after 16 releases and 8 writers held at cut 0, so a diary has at most 8 live, 8 frozen and 16 released writers in G. A rejoin naming a leave on a writer of the same diary that its walk voided (over) is rejected; one naming a leave of a released writer is valid, since that leave still counts. A hole in the history of a diary with an `on: true` join for G (`{ gap: true }` in its marks) may hide the mark that freed or froze a slot, so it sets `incomplete`. The fold returns `frozen` (the frozen writers, released ones included), `released` (the released writers it kept) and `slots` (diary -> `{ live, frozen, released }`, the counts of each diary with an `on: true` join for G, from which nodes derive the room left). `diaryGroupKeys` and the seeder ("Seeder group admission") use the same walk, so a second device of the account and the seeders open the same roots and writers. -**Writers named by a node** (`countedCuts` and `groupCuts` in `member.js`). In a community, an admit, a ban or a demotion that a node writes names every counted writer of the member, a frozen one at its cut, never past it, and never a released one (8 live and 8 frozen fit `MAX_GROUP_CUTS`; 16 released writers would not, and only their block 0, leaves and key blocks count). In a group, an admit, a ban or a demotion names the writers a new post would count on (a probe post past each writer's length) and every frozen writer the fold counts, a frozen writer whose cut is at or below its length included although it takes no new post, each at its length or at the diary's cut when that is lower, and never a released one, as the browser node does (`cutsFor`): a ban or a demotion covers a writer it does not name from seq 0, so leaving a frozen writer out would hide its earlier posts or void its earlier admin acts in one node and not in the other. A leave and an `on: true` grant name the live writers only, so they stay within 8 cuts. A community demotion is never refused for its size (both nodes): past `MAX_GROUP_CUTS` it names the first writers by key and the rest read as cut at 0, while an `on: true` grant past it fails with `COMMUNITY_FULL`. The writers an admission waits for (`unheldCountedWriters`) come from the walk too: a frozen or released writer is awaited (its leaves still count), except, in a private group, one whose newest join is off: an admit never names a released writer, and a replaced boxless one can never be held (seeders and sync refuse it), so waiting for it would hold the admission for good. The fold itself still awaits it ("Limits"). A node that replaces its own writer of a private group whose header box is not the device's (a join before the root arrived; `joinLoaded` in the core and browser nodes) writes, after that writer's join off and before the new writer's join, a `group-cut` on it (`replacedWriterCut` in `group.js`), and never a second one at the same or a higher cut. The cut is 0 for a header with no box: the walk releases it whatever it holds, and nothing past its block 0, leaves and key blocks ever counted in a private group, where a boxless writer is rejected. For a header with a box it is one past the last seq the node holds (1 for a header with a rejoin and nothing else, 0 for a bare header), so nothing it holds stops counting. The join off alone keeps its slot in the walk. With an unknown length (the core not held) no cut is written. Before writing the join off, the cut or the new join, the node steps its diary's own walk with those events (`walkHasRoom` in `group.js`): when the new writer would not take a live slot, it writes nothing and fails with `GROUP_FULL`. A node that replaces its own live writer because the fold no longer takes it (retired by a leave of any device, or naming a CW the gate retired; `joinLoaded` in both nodes) writes the same way, after that writer's join off (when its newest join is on) and before the new writer's join, a `group-cut` on it (`retiredWriterCut` in `group.js`): 0 for a bare header, 1 for a header with a rejoin and nothing else (its block 0 holds the generation), else one past the last seq it holds (a device holds its own core whole), so no record it wrote stops counting and the walk frees its live slot (a cut 0 releases it, a cut >= 1 freezes it, each while there is room). A mark already on that writer does not stop the cut while the walk still has the writer live (the mark came when the walk had no room to freeze or release it): the repeated cut is a new walk event, so the walk can take it now, and it goes at the lowest mark on the writer when that is lower, never higher. A writer the walk already froze or released gets no further cut. Only this device's writer is cut this way: another device's writer is cut only by a disconnect (section 1). With block 0 not held no cut is written. The join off and the cut are steps of the same `walkHasRoom` check: when the new writer would still not take a live slot (8 live, or no room to freeze or release the old one), nothing is written and the join fails with `GROUP_FULL`. A `community.join` that replaces this device's retired CW (`communityJoin` in both nodes) writes the same way, and its join off and cut take part in the same check before anything is written (`COMMUNITY_FULL` with nothing written when the new CW would not take a live slot). The cut goes after the leave the join may append to that CW first (`ownRejoinLeave`), so that leave stays below it, and before the new CW's join. Every per-diary check a node makes before a join (`GROUP_FULL`, `COMMUNITY_FULL`) counts the live slots of its own diary's walk, never stored cores or the earliest 8 joins, and the browser node's `writersBound` (rpc.md 5.42) is that live count. The policy facts of a periodic rekey (`epochFacts` in `group-rekey.js`) skip the `group-sealed` envelopes a frozen or released writer wrote at or past its cut, as the fold drops them. +**Writers named by a node** (`countedCuts` and `groupCuts` in `member.js`). In a community, an admit, a ban or a demotion that a node writes names every counted writer of the member, a frozen one at its cut, never past it, and never a released one (8 live and 8 frozen fit `MAX_GROUP_CUTS`; 16 released writers would not, and only their block 0, leaves and key blocks count). In a group, an admit, a ban or a demotion names the writers a new post would count on (a probe post past each writer's length) and every frozen writer the fold counts, a frozen writer whose cut is at or below its length included although it takes no new post, each at its length or at the diary's cut when that is lower, and never a released one, as the browser node does (`cutsFor`): a ban or a demotion covers a writer it does not name from seq 0, so leaving a frozen writer out would hide its earlier posts or void its earlier admin acts in one node and not in the other. A leave and an `on: true` grant name the live writers only, so they stay within 8 cuts. A community demotion is never refused for its size (both nodes): past `MAX_GROUP_CUTS` it names the first writers by key and the rest read as cut at 0, while an `on: true` grant past it fails with `COMMUNITY_FULL`. The writers an admission waits for (`unheldCountedWriters`) come from the walk too: a frozen or released writer is awaited (its leaves still count), except, in a private group, one whose newest join is off: an admit never names a released writer, and a replaced boxless one is never held by the member sync, which refuses its header (seeders keep it only as a released stub), so waiting for it would hold the admission for good. The fold itself still awaits it ("Limits"). A node that replaces its own writer of a private group whose header box is not the device's (a join before the root arrived; `joinLoaded` in the core and browser nodes) writes, after that writer's join off and before the new writer's join, a `group-cut` on it (`replacedWriterCut` in `group.js`), and never a second one at the same or a higher cut. The cut is 0 for a header with no box: the walk releases it whatever it holds, and nothing past its block 0, leaves and key blocks ever counted in a private group, where a boxless writer is rejected. For a header with a box it is one past the last seq the node holds (1 for a header with a rejoin and nothing else, 0 for a bare header), so nothing it holds stops counting. The join off alone keeps its slot in the walk. With an unknown length (the core not held) no cut is written. Before writing the join off, the cut or the new join, the node steps its diary's own walk with those events (`walkHasRoom` in `group.js`): when the new writer would not take a live slot, it writes nothing and fails with `GROUP_FULL`. A node that replaces its own live writer because the fold no longer takes it (retired by a leave of any device, or naming a CW the gate retired; `joinLoaded` in both nodes) writes the same way, after that writer's join off (when its newest join is on) and before the new writer's join, a `group-cut` on it (`retiredWriterCut` in `group.js`): 0 for a bare header, 1 for a header with a rejoin and nothing else (its block 0 holds the generation), else one past the last seq it holds (a device holds its own core whole), so no record it wrote stops counting and the walk frees its live slot (a cut 0 releases it, a cut >= 1 freezes it, each while there is room). A mark already on that writer does not stop the cut while the walk still has the writer live (the mark came when the walk had no room to freeze or release it): the repeated cut is a new walk event, so the walk can take it now, and it goes at the lowest mark on the writer when that is lower, never higher. A writer the walk already froze or released gets no further cut. Only this device's writer is cut this way: another device's writer is cut only by a disconnect (section 1). With block 0 not held no cut is written. The join off and the cut are steps of the same `walkHasRoom` check: when the new writer would still not take a live slot (8 live, or no room to freeze or release the old one), nothing is written and the join fails with `GROUP_FULL`. A `community.join` that replaces this device's retired CW (`communityJoin` in both nodes) writes the same way, and its join off and cut take part in the same check before anything is written (`COMMUNITY_FULL` with nothing written when the new CW would not take a live slot). The cut goes after the leave the join may append to that CW first (`ownRejoinLeave`), so that leave stays below it, and before the new CW's join. Every per-diary check a node makes before a join (`GROUP_FULL`, `COMMUNITY_FULL`) counts the live slots of its own diary's walk, never stored cores or the earliest 8 joins, and the browser node's `writersBound` (rpc.md 5.42) is that live count. The policy facts of a periodic rekey (`epochFacts` in `group-rekey.js`) skip the `group-sealed` envelopes a frozen or released writer wrote at or past its cut, as the fold drops them. ### Folding: `buildGroup` -`buildGroup({ root: { key, entries }, writers: [{ key, entries, diary }], joins: Map, community, keys, marks, confirms, now })` is a pure fold (`marks`, optional, is `Map` from `groupCutHistory`, "Diary cuts on own writers" and "Creator marks on group roots"; `buildCommunity` takes `marks` too; `confirms`, optional, is `Map` from `groupConfirmHistory` (`dseq` the confirmation's own seq in the diary), "Confirmed leaves"; `buildCommunity` takes `confirms` too); `community` (`null`, or the parent's input `{ root: { key, entries }, writers, joins }`, exactly `buildCommunity`'s input) is defined under "Communities contain groups", and `keys` (optional, `{ writers, unseal, onKey? }`: the reader's means to open epoch keys, and the hook that hands a node the accepted keys) and the private-only results `sealed`, `epoch`, `rekeyPending`, `entitled` and `removals` under "Private groups". It never throws and gives the same result for any delivery order of the same blocks. The first valid copy of a seq is retained. It returns `kind`, `name`, `join`, `visibility`, `creator`, `admins: Set` (the community's creator and admins included, below), `community` (the parent's fold result when C authorizes G, else null), `orphan` (true when C folds but does not authorize G, below), `members: Map`, `posts: [{ writer, diary, seq, at, text, replyTo, hidden }]` in causal order, `requests` (requested diaries), `future` (skipped post refs), `last` (the newest node in causal order, a rejoining writer's block 0 included, never a reaction, as `{ writer, seq }`, or null), `bans: Map` (the latest valid ban of every diary banned now: the `lifts` an unban must name), `leaves: Map` (the causal latest leave of every diary that has left now: the `rejoin` a new writer must name), `lastLeave: Map` (the causally latest leave of every diary that ever left, kept after a rejoin, as in a community), `liveWriters: Set` (the counted writers that are live now: not retired, not banned, not cut by their own diary, and, under the gate, with a live CW in C; the root only when `rootLive`), `authorityWriters: Set` (the live writers that hold moderation power now, for nodes to pick a writer that can sign an admin record), `cuts: Map` (each counted writer its own diary cut, with the lowest cut, "Diary cuts on own writers"; `buildCommunity` returns it for CWs), `cutUnknown: Set` (the diaries binding a counted writer whose marks as read have a hole, same section; `buildCommunity` returns it too), `writers: Set` (the writer cores the fold counted; the root only when bound), `memberInvite` (whether members may invite, "Members may invite") and `incomplete`. A missing named writer sets `incomplete: true` without blocking other writers. An invalid or absent root header yields the empty incomplete result, and so does a root whose header kind is `community` (a community is folded by `buildCommunity`, never by `buildGroup`). An unbound root (no `on: true` join for `writer = G` in the creator's diary) sets incomplete, carries no authority and contributes no posts: only its header is read, and the header's diary is an ordinary diary (not listed in `admins` or `members` unless it has counted writers of its own); `creator` in the result still echoes the header. A node whose own writer is outside `writers` (beyond the cap) refuses to write (`limite do grupo atingido`) and never reports `member` for it. +`buildGroup({ root: { key, entries }, writers: [{ key, entries, diary, signed? }], joins: Map, community, keys, marks, confirms, now })` is a pure fold (`marks`, optional, is `Map` from `groupCutHistory`, "Diary cuts on own writers" and "Creator marks on group roots"; `buildCommunity` takes `marks` too; `confirms`, optional, is `Map` from `groupConfirmHistory` (`dseq` the confirmation's own seq in the diary), "Confirmed leaves"; `buildCommunity` takes `confirms` too); `community` (`null`, or the parent's input `{ root: { key, entries }, writers, joins }`, exactly `buildCommunity`'s input) is defined under "Communities contain groups", and `keys` (optional, `{ writers, unseal, onKey? }`: the reader's means to open epoch keys, and the hook that hands a node the accepted keys) and the private-only results `sealed`, `epoch`, `rekeyPending`, `entitled` and `removals` under "Private groups". It never throws and gives the same result for any delivery order of the same blocks. The first valid copy of a seq is retained. It returns `kind`, `name`, `join`, `visibility`, `creator`, `admins: Set` (the community's creator and admins included, below), `community` (the parent's fold result when C authorizes G, else null), `orphan` (true when C folds but does not authorize G, below), `members: Map`, `posts: [{ writer, diary, seq, at, text, replyTo, hidden }]` in causal order, `requests` (requested diaries), `future` (skipped post refs), `last` (the newest node in causal order, a rejoining writer's block 0 included, never a reaction, as `{ writer, seq }`, or null), `bans: Map` (the latest valid ban of every diary banned now: the `lifts` an unban must name), `leaves: Map` (the causal latest leave of every diary that has left now: the `rejoin` a new writer must name), `lastLeave: Map` (the causally latest leave of every diary that ever left, kept after a rejoin, as in a community), `liveWriters: Set` (the counted writers that are live now: not retired, not banned, not cut by their own diary, and, under the gate, with a live CW in C; the root only when `rootLive`), `authorityWriters: Set` (the live writers that hold moderation power now, for nodes to pick a writer that can sign an admin record), `cuts: Map` (each counted writer its own diary cut, with the lowest cut, "Diary cuts on own writers"; `buildCommunity` returns it for CWs), `cutUnknown: Set` (the diaries binding a counted writer whose marks as read have a hole, same section; `buildCommunity` returns it too), `writers: Set` (the writer cores the fold counted; the root only when bound), `memberInvite` (whether members may invite, "Members may invite") and `incomplete`. A missing named writer sets `incomplete: true` without blocking other writers. An invalid or absent root header yields the empty incomplete result, and so does a root whose header kind is `community` (a community is folded by `buildCommunity`, never by `buildGroup`). An unbound root (no `on: true` join for `writer = G` in the creator's diary) sets incomplete, carries no authority and contributes no posts: only its header is read, and the header's diary is an ordinary diary (not listed in `admins` or `members` unless it has counted writers of its own); `creator` in the result still echoes the header. A node whose own writer is outside `writers` (beyond the cap) refuses to write (`limite do grupo atingido`) and never reports `member` for it. **Authority and layers.** @@ -1954,7 +1954,9 @@ A CW record (admit, ban, hide or invites) whose `group` names a key outside C's **Diary cuts on own writers** (owner decision, #350). A diary can cut any writer of its own, for instance the writer of a lost device: nothing that writer wrote at or past the cut counts, except its leaves and key blocks (below), and a cut is final ("Corte é final", owner decision): no record of another diary brings a dropped record back, whatever position it names (a demotion, ban or admit of d listing w at any length, or a basis ref on w). For a counted writer w (never G's root, never a community root) bound by diary d, `cut(w)` is the lowest `cut` among the valid `group-cut` marks of d for this group (or community) whose `writer` is w. Only d's own marks count: a mark in another diary naming w, or a mark of d naming a writer d does not bind, does nothing. -Binding drops the records of w at seq >= cut(w), as if w's core ended there, with three exceptions that always stay, whatever the cut: block 0 (the header and a rejoining writer's rejoin node), every `group-leave` (such a cut never undoes a leave, never revives a diary that left and never breaks a rejoin naming that leave, so `removals` and `leaves` are unchanged), and every `group-key` block. A leave written after the mark counts like one written before it (owner decision, 16/11/2026): a leave only takes, never gives, and a leave made on a lost device shortly before it was lost is not undone; the accepted limit is that a stolen device already offline can still take the person out of a group or an admin role, and the person asks to join again. A cut at 0 is the same rule with nothing kept below the cut (owner decision, 16/11/2026): w keeps block 0, every leave and every key block, stays a candidate and is awaited ("Limits", slot walk), and a rejoin naming one of its leaves stays valid. This holds for every mark, not only for honest nodes: whoever can sign d's diary (a thief with the unlocked identity included) can cut the owner's other device at 0 and free its slot, but never undo its leave. The records stop at the lowest cut among every valid mark of d on w the reader holds, even one the slot walk ignored for the slot (written before the join it reads, or held by a reader whose hole hides that join): the walk decides the slot, never the records. A key block past the cut stays valid for reading only: its keys open history as before, but it never sends and adds no recipient to rule 4 of "Eligibility". Everything else past the cut (posts, reactions, topics, polls, choices, events, hides, admits, bans, admin grants, meta, invite flags, sealed envelopes) does not count, and a reference into it is ignored like any reference past a core's end. Every later rule (order, authority, `lastSeq`, keys) sees the shortened log, except that a G key block whose `auth` cites a cut CW is judged against the CW's held length, so the history it handed out stays readable; it still gives no sending authority, since a cut CW is not in C's `authorityWriters`. A cut writer is in neither `liveWriters` nor `authorityWriters`. Its slot follows the slot walk ("Limits"): a cut >= 1 freezes it (still counted, after the live writers of its tier under the 512 cap), a cut 0 releases it, each while there is room; with no room it keeps its slot, cut. The fold returns `cuts` (writer -> cut). The result is the same for any order or arrival of the marks, before or after a rotation. +Binding drops the records of w at seq >= cut(w), as if w's core ended there, with three exceptions that always stay, whatever the cut: block 0 (the header and a rejoining writer's rejoin node), every `group-leave` (such a cut never undoes a leave, never revives a diary that left and never breaks a rejoin naming that leave, so `removals` and `leaves` are unchanged), and every `group-key` block. A leave written after the mark counts like one written before it (owner decision, 16/11/2026): a leave only takes, never gives, and a leave made on a lost device shortly before it was lost is not undone; the accepted limit is that a stolen device already offline can still take the person out of a group or an admin role, and the person asks to join again. A cut at 0 is the same rule with nothing kept below the cut (owner decision, 16/11/2026): w keeps block 0, every leave and every key block, stays a candidate and is awaited ("Limits", slot walk), and a rejoin naming one of its leaves stays valid. This holds for every mark, not only for honest nodes: whoever can sign d's diary (a thief with the unlocked identity included) can cut the owner's other device at 0 and free its slot, but never undo its leave. The records stop at the lowest cut among every valid mark of d on w the reader holds, even one the slot walk ignored for the slot (written before the join it reads, or held by a reader whose hole hides that join): the walk decides the slot, never the records. A key block past the cut stays valid for reading only: its keys open history as before, but it never sends and adds no recipient to rule 4 of "Eligibility". Everything else past the cut (posts, reactions, topics, polls, choices, events, hides, admits, bans, admin grants, meta, invite flags, sealed envelopes) does not count, and a reference into it is ignored like any reference past a core's end. Every later rule (order, authority, `lastSeq`, keys) sees the shortened log, except that a G key block whose `auth` cites a cut CW is judged by `auth.seq` against the CW's signed length ("Sparse copies", below), so the history it handed out stays readable, from a copy trimmed at the cut too; it still gives no sending authority, since a cut CW is not in C's `authorityWriters`. A cut writer is in neither `liveWriters` nor `authorityWriters`. Its slot follows the slot walk ("Limits"): a cut >= 1 freezes it (still counted, after the live writers of its tier under the 512 cap), a cut 0 releases it, each while there is room; with no room it keeps its slot, cut. The fold returns `cuts` (writer -> cut). The result is the same for any order or arrival of the marks, before or after a rotation. + +**Sparse copies** (#350). A seeder keeps a cut writer sparse ("Seeder group admission"): its blocks below the cut, then only its leaves and key blocks, so a reader may hold a writer up to a hole and some blocks past it. Every reader (the core node, the browser node and the account service) reads a writer up to its first missing block and, past it, only the `group-leave` and `group-key` blocks it holds, up to `MAX_LOG_BLOCKS` (`readHeldBlocks` in `store.js` and `node-store.js`), and hands the fold each copy's `signed`: the core length its writer signed, against which Hypercore verified every block held. Roots are still read as a prefix. The fold takes, of a writer, the records below the first seq no copy holds and, past that hole, only its leaves and key blocks, and only when the hole is at or past the writer's cut (`max(cut, 1)`, so a released writer needs its block 0): a sparse copy of a cut writer folds exactly like the whole copy, and a partial copy, cut or not, folds as the prefix it always was. A block past a hole below the cut never counts, since the whole copy would count the records missing before it. A cut writer's signed length is the largest `signed` among its copies; a key block whose `auth` cites a cut CW is valid for reading when `auth.seq` is at most that length (or the CW's held length, when greater), so a CW trimmed at its cut reads the history the whole copy reads. The browser node's fold memo stamps the runs of blocks held past the contiguous length, read from the bitfield (no stamp, so no memo, past 4096 runs), so a block that arrives past a hole folds again. A `{ gap: true }` entry cuts nothing (the root rule under "Creator marks on group roots" keeps its own gap handling): a hole only delays a mark until it fills, and treating it as a cut would let any hole silence every device of the diary. The fold reports such diaries in `cutUnknown` (only those binding a counted writer), and the slot walk's gap rule ("Limits") sets `incomplete`. A node must not take the fold as complete about cuts for them: it keeps fetching that diary before deciding anything that depends on a writer of it being uncut, for instance before rotating away from it or treating its device as current. @@ -1984,7 +1986,7 @@ Late blocks can change the causal order of posts already displayed. The RPC surf A community C is the parent of groups. C is the key of a single-writer root core whose block 0 is a `group-header` with `kind: 'community'` (`community`, `member` and `box` null, `visibility` `public`, "Cores and headers"; `diary` is the community creator). Unlike a group root, C has writer cores: each device of a member writes a **community writer** (CW), a `group-writer` core with `group` = C and `member: null`, bound to its diary exactly like any writer (the join `{ group: C, writer: CW, on: true }` and a `rejoin` that names a `group-leave` of the same diary inside C). A group G has exactly one parent, the `community` of its header, immutable. C lists its groups in a catalogue made of `group-create` records ("Community-core records"); the first group of a community, created with it, is called Geral. Readers decide the kind from the root's own header. - **Bound root.** As for a group root, the header's `diary` is only a label until that diary binds the root with a join `{ group: C, writer: C, on: true }`. An unbound community root keeps only its header: it has no creator, no admins, no members and no catalogue, gives no power in any group, and `buildCommunity` returns `incomplete: true`. -- **`buildCommunity`.** `buildCommunity({ root: { key: C, entries }, writers: [{ key, entries, diary }], joins: Map, now })` is the input of `buildGroup` without `community` and `keys`. The community fold reads the diaries' cuts (`marks`), like the group fold: the core node's `loadCommunity` and sync catalogue (`heldCatalogue`) and the browser node's `foldCommunity` and sync catalogue (`foldedCatalogue`) pass them. It runs the group fold on C's logs in community mode: the same binding (`readJoins`), per-diary cap, 512 cap with the same tiers, rejoin check, causal order, leave withdrawal, root admin layer (its cuts are over CW keys), admit and ban layers, shield and results `members`, `requests`, `bans`, `leaves`, `last` and `writers`. The differences are: +- **`buildCommunity`.** `buildCommunity({ root: { key: C, entries }, writers: [{ key, entries, diary, signed? }], joins: Map, now })` is the input of `buildGroup` without `community` and `keys`. The community fold reads the diaries' cuts (`marks`), like the group fold: the core node's `loadCommunity` and sync catalogue (`heldCatalogue`) and the browser node's `foldCommunity` and sync catalogue (`foldedCatalogue`) pass them. It runs the group fold on C's logs in community mode: the same binding (`readJoins`), per-diary cap, 512 cap with the same tiers, rejoin check, causal order, leave withdrawal, root admin layer (its cuts are over CW keys), admit and ban layers, shield and results `members`, `requests`, `bans`, `leaves`, `last` and `writers`. The differences are: - Only records with `group: null` act in C. Records with `group !== null` (admit, ban, hide) are nodes in C's causal order and change nothing in C; a `group-hide` never sets a hide in C. - **Members may invite in C.** C's `memberInvite` is the `on` of the last `group-invites` record with `group: null` in C's causal order that counts as an administrative node of C: on C's root, the creator's while the root has its powers (root seq below `rootCut`); on a CW, the creator's diary or a diary whose C admin layer is on and C ban layer off at that CW and seq, the CW counted, not retired (no authority after leaving, past records included) and the record not withdrawn by a leave (`authorizedC` in `lib/group.js`, the gate of a `group: null` admit). A returning creator's record on a creator CW also counts when it passes the repair gate ("Community repair after the creator's return"), as for a CW meta. A member's record never counts. Absent, it is `false` (also for an unbound or empty C). A `group-invites` naming a group changes nothing in C. - A community has no posts: `posts: []`, `future: []`, `sealed: 0`, `epoch: null`, `rekeyPending: false`, `entitled: []`, `removals: []`. `join` and `name` come from the header and the last not-withdrawn `group-meta` by root seq, overridden by the latest counted CW meta ("Community repair after the creator's return"). @@ -2082,7 +2084,7 @@ A private group (`visibility: 'private'` in the root header, kind `group`) is a **Distributors.** A `group-key` block counts for reading when its writer was authorized to distribute at that writer/seq: a bound writer of the creator (root bound), or a writer that passes the ordinary authority check for administrative actions there (group admin layer, ban layer off), or, through community authority, a writer whose block carries a valid `auth` (below). Leave withdrawal does NOT apply to reading, neither to the `group-key` block nor to the grant that gave its writer authority (the distributor check reads the root's `group-admin` records and C's admin layer without leave withdrawal), so an admin who later leaves or is demoted, or a creator whose leave withdraws the grant, never takes back the epochs already handed out and history stays readable. For reading, a demotion cuts only the writers it lists, at their listed cut (a writer it omits keeps the read validity of its blocks, though it is cut at 0 for everything else), and a root demotion never reaches a block that sorts before it in causal order, whatever its cuts; grants still give nothing to unlisted writers and bans still apply by their cuts. So no demotion, honest or not, makes history unreadable. Sending keeps the strict layer: a first block counts for Eligibility, coverage and the key-holding and power records of "Removals" only when its writer passes the ordinary check with demotions cutting every writer, so no member ever sends under the epoch of a writer that lost power. That check also applies leave withdrawal to the grants: a grant that a creator's leave withdrew keeps the blocks it authorized readable but gives no sending power. A block valid for sending must also come from its diary's current generation: when a leave of that diary sorts before the block in causal order, its authority must come from a grant that saw every leave of that diary sorting before the block ("Entitled writers"; for the creator, a writer whose rejoin names the latest such leave). A block that sorts before every leave of its diary keeps its authority, so the epochs an admin opened before leaving stay valid history, and a block that covers a leave has that leave as an edge, so it needs a grant written after the leave. The writer that left, any other writer of that diary from before the leave (whatever grant or community power still lists it) and a creator's root after the creator left keep reading power only, so a block they write after a rejoin, wherever its `at` sorts it, never becomes an epoch anyone sends under, never counts as a recipient or authority record and is never reported by `onKey`. A rejoined distributor regains sending power on its new writer once a grant that saw its leave lists that writer. Other blocks are ignored. By owner decision, the creator of C and the admins of C (`C.admins`) distribute and receive keys in every private group of C through community authority, like group admins, while they hold that power; losing it is an admin change (below). For sending, an epoch must also pass "Eligibility". -**Community distributors (`auth`).** A `group-key` block K in a G writer w of diary d, when d is not G's creator and no G grant covers (w, seq), counts through the community only when `K.auth` is not null and names a position `{ writer: cw, seq: s }`, where `s` (an integer, at least 1) is the length of the distributor's own CW when it wrote the block (the next position in that CW, like a cut), such that, with C's fold, `cwOk` holds (`auth` must cite the writer's own CW): `C.cwDiary.get(cw) == d` (C's root for C's creator) and `s` is at most the held length of that core (the length it cites exists here). Reading (`dist`) needs `cwOk` and, for C's creator, the root bound, or, for anyone else, some counted `on: true` root grant of C that lists cw with a cut at or below s (demotions, C bans and leave withdrawal are ignored for reading, as history must never become unreadable: a grant and a later demotion can both cut at the same length when the admin wrote nothing in its CW in between). The community gate never withdraws a key block's read validity either. Sending (`send`) needs the strict layer: d's grant in C's current admin layer on at (cw, s) and C's ban layer off there, `C.admins.has(d)`, `live(w)`, and the cited CW holding C authority now (in C's `authorityWriters`: a CW retired, banned per CW or demoted never sends, even when its diary is still a C admin through another device). `auth` is ignored (as null) on a block whose writer holds G authority, so G authority wins; a block with `auth: null` from a diary without G authority is not counted. `auth` is a validity citation, not an ordering edge: an edge into another core's chain would let a CW position pin a block's place in G's order. Nodes write `auth = { writer: ownCW, seq: ownCW.length }` when they distribute through community authority, and `null` otherwise. Citing the length matters: a grant's cut is the CW's observed length L, so an admin promoted at L that wrote nothing in its CW since cites L, which the grant covers; a demotion's cut M turns the strict layer off from M on, so a demoted admin's blocks are never sent under. A demoted admin can still write blocks that read as valid by citing an old position; nobody sends under them, and only its own posts could use them, which a community ban or a hide covers. Everything else (kid, commit, tag, basis, rekey-pending, `onKey` only for full epochs) is unchanged. +**Community distributors (`auth`).** A `group-key` block K in a G writer w of diary d, when d is not G's creator and no G grant covers (w, seq), counts through the community only when `K.auth` is not null and names a position `{ writer: cw, seq: s }`, where `s` (an integer, at least 1) is the length of the distributor's own CW when it wrote the block (the next position in that CW, like a cut), such that, with C's fold, `cwOk` holds (`auth` must cite the writer's own CW): `C.cwDiary.get(cw) == d` (C's root for C's creator) and `s` is at most the held length of that core (the length it cites exists here; of a cut CW, the length its writer signed, "Sparse copies"). Reading (`dist`) needs `cwOk` and, for C's creator, the root bound, or, for anyone else, some counted `on: true` root grant of C that lists cw with a cut at or below s (demotions, C bans and leave withdrawal are ignored for reading, as history must never become unreadable: a grant and a later demotion can both cut at the same length when the admin wrote nothing in its CW in between). The community gate never withdraws a key block's read validity either. Sending (`send`) needs the strict layer: d's grant in C's current admin layer on at (cw, s) and C's ban layer off there, `C.admins.has(d)`, `live(w)`, and the cited CW holding C authority now (in C's `authorityWriters`: a CW retired, banned per CW or demoted never sends, even when its diary is still a C admin through another device). `auth` is ignored (as null) on a block whose writer holds G authority, so G authority wins; a block with `auth: null` from a diary without G authority is not counted. `auth` is a validity citation, not an ordering edge: an edge into another core's chain would let a CW position pin a block's place in G's order. Nodes write `auth = { writer: ownCW, seq: ownCW.length }` when they distribute through community authority, and `null` otherwise. Citing the length matters: a grant's cut is the CW's observed length L, so an admin promoted at L that wrote nothing in its CW since cites L, which the grant covers; a demotion's cut M turns the strict layer off from M on, so a demoted admin's blocks are never sent under. A demoted admin can still write blocks that read as valid by citing an old position; nobody sends under them, and only its own posts could use them, which a community ban or a hide covers. Everything else (kid, commit, tag, basis, rekey-pending, `onKey` only for full epochs) is unchanged. **Reading.** `buildGroup` takes an optional `keys`: `null` or `{ writers: Set, unseal(sealed), onKey? }`. `writers` are the reader's own writer keys in this group (current and retired writers, and G when the reader's device created the group: they all share the device's box key). `unseal(sealed)` takes one `sealed` base64 string and returns its raw 48-byte plaintext `epochKey || tag` (`crypto_box_seal_open` with the device key pair) or null; it never checks the tag. The fold requires exactly 48 bytes, checks the tag against the block's G and kid ("`group-key`"), copies the 32-byte key and zero-fills both the 48 bytes and the buffer `unseal` returned. The fold calls `unseal` only on entries naming one of `writers` inside counted `group-key` blocks; an epoch whose first block is valid only for reading (or withdrawn) is opened only when some counted, not withdrawn envelope names its kid, since nothing else could be read under it; an opened key counts only when `keyCommit(key, G, kid)` equals the epoch's first-block `commit` (compared in constant time); an entry that fails is ignored and later entries and blocks are still tried, so for each kid the fold keeps the committed key from the first block in causal order that opens to it. It never puts a key in its result. The optional `onKey(kid, key, recipients)` is called once per epoch the fold accepted and opened whose first block is valid for sending and not withdrawn by a leave (after the tag and commitment checks, in causal order of the first block; an epoch valid only for reading, such as a departed or demoted distributor's, stays readable inside the fold but is never reported, so it is never handed on), with a `{ writer, seq }` copy of the kid, a COPY of the 32-byte key that the caller owns and must zero-fill, and `recipients`, the sorted writer keys listed by every block of that epoch valid for sending and not withdrawn by a leave; an error it throws is ignored and changes nothing. A node gets the keys it may hand on (history, rotation, sealing) only from this hook, never by recomputing distributors itself. An opened envelope becomes its inner record at the envelope's `{ writer, seq }`, with the inner record's ordering edges plus the envelope's `kid` edge (so it never sorts before its epoch's first block), and with the authority, leave withdrawal and membership generation (the leaves of its diary before it) the envelope had in the plaintext pass below; an envelope that stays sealed is a node with only its writer edge and its `kid` edge, and is never shown. An opened `group-react` is the exception: its envelope stays that sealed node and the reaction is folded beside the order ("Reactions"), without counting in `sealed`. The fold runs twice. The plaintext pass keeps every envelope sealed and alone decides membership and moderation (`members`, `requests`, `admins`, `bans`, `leaves`, `community`, `writers`) and the epoch machinery (Distributors, Removals, Entitled writers, Eligibility); the opened pass, under that same authority, only places content (posts, hides, reactions, the name) and orders it, so decrypted `seen`, `replyTo` and `target` edges never change who is a member or who may act. The fold is pure and order independent for a given `keys`; two readers with different keys may see different posts and order, never different membership. The name of a private group is the `name` of the last opened, valid, not withdrawn `group-meta` in the root by root seq; a sealed meta whose `join` is not `approval` is skipped, and `join` is always `approval`. Without a key the name is null. @@ -2171,7 +2173,7 @@ Mirrors "Seeder media admission" (section 5) with the same invariants, plus the - **Root G.** Admitted by the same steps as a writer (including quarantine, persistence before the open and the held blocklist) when its block 0 is a valid `group-header` and some served, gated, unblocked diary has a live join for G (the creator's join with `writer == group` is the normal path). **Root binding.** The header's diary counts as the root's creator (and as support) only when that diary is served, not dropped, not blocked and has its own `on: true` `group-join` for writer = G. Otherwise the root is supported only by real member joins, and `/index/group/` answers `creator: null` (also during the startup window before the creator's scan). **Support:** a served root stays while at least one served, unblocked diary keeps it: the creator if it bound the root, a diary with a bound writer in it, or a live join. Withdrawing every join does not drop a root that still has a bound writer. At startup a root does not survive just because its creator's diary is kept: decisions that find a root unsupported are deferred until the startup scans of the served diaries end, and it is judged after them; an unsupported root is then dropped with reason `unbound`. **Blocked or revoked creator:** a served root is purged for a blocked diary, or one whose certificate issuer is revoked, only when that diary BOUND the root (`bound: true`: its header diary has its own `on: true` join for writer = G). Reason `blocked` (also during the startup window), the root stays refused at admission and at startup while its creator stays gone, even while members support the group. A root whose header merely names a blocked diary that never bound it has no creator and stays while members support it. A creator that is both blocked and from a revoked issuer has that issuer recorded too (`revokedCreators`, and `goneRoots` for its bound roots), and a stored issuer is never replaced by null: lifting only the block keeps its bound roots refused, also after a restart, while the issuer stays revoked. A creator dropped or refused for a revoked issuer is remembered with that issuer (`revokedCreators`, a moderation outcome, also across restarts) and judged against the CURRENT revoked-issuer set: adopting a list that lifts the revocation lifts it. An announce of a revoked-issuer certificate is not remembered at all, not even in memory: only a diary's own block-0 verdict (refused at startup, or dropped as revoked-issuer while served) feeds `revokedCreators`, a `goneRoots` issuer and the refusal of a root at admission for a revoked creator, so a root whose creator was never served on this seeder is not refused for a revoked issuer (the root only names that diary). A repeated revoked announce of the same key from one peer counts as a bad announce; the first does not. A purged root is remembered with its creator and, for a revocation, its issuer (`goneRoots`) and refused again while that creator stays gone; lifting the block or revocation lets the same join admit it, and a root admitted again after that keeps `bound`. A diary admitted with a verifying certificate clears its stale revoked entries (`revokedCreators` and the `goneRoots` entries naming it with an issuer). A root whose header names a currently blocked diary, or a diary whose known issuer is revoked, is refused at admission even without a purge record (a root admitted before the block stays until its binding says otherwise). Roots bound before this version get `bound` only after their creator's next scan. A root its creator bound stays listed on `/index/group/` (and in the writer pages) under that creator diary while members keep it served after the creator is dropped for a reason other than moderation (quota, fork, seeder full), also after a restart; `header.diary` stays `null` then, so no creator authority is shown. Members' own writers stay under the rules above. **Issuer kept with the binding.** A bound root's binding keeps the issuer of its creator's block-0 certificate (`issuer`, taken at the creator's scan). When that issuer is revoked under the current list and the creator is not served, the bound root is purged with reason `revoked-issuer` and remembered like any revoked-issuer drop (`revokedCreators`, `goneRoots`), also at startup and also when the creator left first for another reason, such as its quota, so a member that keeps the root served cannot outlast the revocation. **Header fields kept.** While any core of group G is served or being admitted, the seeder keeps the verified header fields of its root (`kind`, `visibility`, `join`, `community`, `member`, and `name` for a public root) in `groupHeaders`, also after the root was purged or refused or its block 0 is withheld, and drops them once no core of G is left or G is blocked; past the bound the record is not kept. A persisted record without `community` and `member` is a protocol v1 leftover and is dropped at start. - **Protocol v2 admission and scan.** A header whose validator returns `legacy` is `header-invalid` (no new wire reason), and the startup re-gate purges every stored v1 root and writer with that reason. A group root must carry a hex64 `community`; the seeder never checks a writer's `member` or any authorization (the fold does). CWs are admitted like any writer and take part in the writer pages. For a served community root the index folds `group-meta`, `group-admin` and `group-create` from the root only (creates are indexed from C's root alone; a CW is not scanned for them, `group-create` being `root-only` there). A public group root is scanned for `group-meta` only (one scan at a time across group roots, skipping blocks without the bytes `group-meta`) and a private root is not scanned. **Denial bounds.** `revokedCreators` and `goneRoots` never evict an entry still in force: when a map reaches 4096 entries the lifted ones go first, and `admitted.json` may hold up to 8192 entries in each. While 4096 or more denials are in force in either map, or after a denial found no room (past 8192), the seeder refuses every group root with the group-rejected reason `denials-full` and logs `{ event: 'warning', reason: 'denials-full' }`, so a purged root cannot return for lack of memory. -- **Cut writers** (#350). For now a writer its diary cut (frozen, released, or live with a cut once the frozen slots or the releases are spent) is held whole, like any writer: its full log under the whole-log quota (at admission, after the open and on every append) and the caps above (32 cores per diary and group, 1024 per diary). The fold counts only block 0, the leaves and the key blocks past a cut, but every node reads a writer from block 0 and stops at its first missing block, so a seeder that cleared the other blocks past the cut would hand a fresh reader a copy that folds without the leave. Keeping only what the fold counts waits for client nodes that read such sparse copies. The cut is not persisted with the binding: the diary's scan recomputes cuts and releases. +- **Cut writers** (#350, owner decision). A writer its diary cut (frozen, released, or live with a cut once the frozen slots or the releases are spent: `groupWalkCut` over the slot walk) keeps its blocks below the cut, block 0 always, and from the cut on only its `group-leave` and `group-key` blocks, the ones the fold still counts there and readers read past a hole ("Diary cuts on own writers", "Sparse copies"). A released writer (cut 0) is a stub: block 0, its leaves and its key blocks. Its download stops at the cut; the blocks past it, up to `MAX_LOG_BLOCKS`, are fetched `CUT_CHUNK` = 256 at a time, read for their type only (`decodeBlock`, never validated) and cleared unless kept. Hypercore keeps the tree nodes the proofs of the kept blocks need, so a reader verifies each against the signed length. The binding in admitted.json carries `cut`, `clearedFrom` and `clearedTo` (the range past the cut already read) and is persisted before a chunk is cleared; the next serve reads that range again and clears what it still holds beyond the kept blocks, so a crash in between leaves nothing extra for good. A chunk ends once every block its synced peers advertise there is held (`advertisedHeld`, "Completion goals"), even when they lack some of it: a seeder fed only by peers that keep the writer sparse (another seeder, or after a `quota` drop that a lower cut lifted) still reads past the first chunk. What it holds there is kept or cleared as above, and a chunk read that way joins the binding's unsure range (`unsureFrom`, `unsureTo`, within the range already read), read again chunk by chunk at intervals doubling from 1 s to 5 minutes, and at once when a peer that joins the core advertises a block there the seeder lacks (checked every second from its join, up to 10 times, once it has synced and answered the want the seeder sends it for that range); a peer offering nothing there, such as a reader that connects or one reconnecting in a loop, leaves the backoff as it is. A chunk read again whole leaves the range, so a leave or key block those peers lacked comes from a fuller peer. The quota counts what is kept: at admission and at startup, the blocks below the cut (`min(max(cut, 1), length)` against `maxLogBlocks`); while the whole log is over the quota, the blocks below the cut and those kept past it, and their bytes, against `maxLogBlocks` and `maxLogBytes`, counted as they are read. A writer past it is dropped with `quota` and its join is refused as at admission. The caps above count a stub like any writer. The diary's scan gives the cut: a lower one applies at once (also to a core still opening), a higher one or none only once the scan caught up with the diary (a cut restored from admitted.json whose record was deleted, for example), and then the download reaches the new cut, or the whole log, so what clearing removed is fetched again. A cut restored at startup clears nothing new until the scan confirms it. A join refused before (`quota`, `header-invalid`) is tried again once per lower cut. A writer of a private group whose header has no box (the one a node replaced, "Writers named by a node") is `header-invalid` while counted and kept as a stub once released, so its block 0 stays available to readers, which reject it by its header ("Limits"); if its release goes away it is judged again and dropped. - **Drops** (purge before un-listing, runtime order of media (e), each logs `{ event: 'group-rejected', key, diary, group, reason }`): the diary is dropped or blocked (D's writers go with it, with `blocked` or the diary's own reason; the root stays while supported), the root key is in the blocklist snapshot (`group-blocked`, for the root and every writer of that group; the blocklist apply loop also walks the group map, because it otherwise walks only `_served`), a fork (`fork`), the quota (`quota`), an unsupported root (`unbound`), a root whose creator (the header diary, when it bound the root) is blocked or has a revoked issuer (`blocked`) or a failed header re-check (`header-invalid`). NEVER on a `group-leave`, never on a join turned off and never on a cut: a cut or released writer stays held whole ("Cut writers"). - **Retention.** Index withdrawal and membership end are distinct, as defined above. Admitted writer and root cores stay served and listed across either event (the index only marks the writer `left`), preserving the moderation records of departed members. - **Startup** re-gates listed group cores like listed media cores (a core whose binding is gone, or whose root is blocked, is purged first), applying the same per-diary counts to persisted bindings (32 per group); for bindings written before the cap that count follows iteration order, not earliest join. A cut writer, a released one included, is restored whole (quota on the whole log), and the scans recompute cuts and releases from the diaries. `seed list` prints the group cores indented under their diary (`grupo (grupo ) blocos bytes`), `seed drop` purges them as auxiliaries, and `stats` adds `groupCores`. @@ -2213,7 +2215,7 @@ Added to section 7 "Seeder index route", under the same caps, Host check, 405 an - **Round 13, core node.** (1) A new writer key of a group is refused (`group-skipped` reason `quota`) while the group's foreign writer cores in the admission table plus those stored on disk, including writers kept after a quota stop, reach `MAX_GROUP_WRITERS` (`runSync` option `maxGroupWriters`; the group root is no writer slot and is never refused by this cap); a stored writer keeps its place, and a refused key is gated again once a place frees. (2) Block-0 divergences of group writers become one hard-error line `bloco 0 divergente em escritores de grupo: ()` (the result is still not ok), and a run keeps at most 32 hard-error strings. - **Membership and purges.** A fetched historical join block proves binding only. Current membership comes from the writer fold above. A stored writer core is purged only for an ACCEPTED writer (one that passed the gate): when D is blocked or its issuer is revoked, on a fork, or when its local block 0 differs from the gated one. A bad hint from a seeder never purges an unaccepted core. A blocked group root fails and purges every foreign core of that group with reason `blocked`, in the core node and in the browser alike; the member's own cores are never touched, and what is stored can still be read locally until it is purged. - **Binding diary release (core node).** A group writer's binding diary is downloaded and kept in the main store only while something needs it: a follow holds it, a writer in the admission table names it, or a stored foreign writer or root names it (the census is rebuilt from disk at start). When a writer is purged, retires or is evicted and nothing else needs its diary, the node stops downloading the diary and purges it from the main store. A writer stopped for quota keeps its diary (and its blocks). At run start a binding diary recorded in `group-certs.json` that no stored core names and that no follow holds is purged. At core node start, a member of any group also purges an unfollowed, unlisted diary unless a stored group writer or root names it or `group-certs.json` records it; that cleanup always rebuilds the census first, also when `groups.json` and `group-certs.json` are empty, and never purges a group core this node can write. A delete of a `group-join` block has no effect (section 2 "Deletion"): a held one keeps its writer bound and downloaded. One the node does not hold is fetched once (the binding diary is group-bound), and the gate waits a bounded time for it; only when the block is still missing after that wait does the core node drop the writer from the run (`group-skipped` reason `withdrawn`), without purging its blocks, and the group goal stops waiting for it. At browser node start, a stored replica that a stored group core binds (or that the run holds as a binding diary) stays indexed and is never purged by the unfollowed-replica cleanup. -- **Completion goals** (`--once` and the browser's once runs). Each group goal runs once per run and is met when (a) every writer page of every joined group was fetched (not `incomplete`) or the channel timed out, (b) every admitted foreign writer, and the diary that binds it up to its join, reached the length its seeder advertised, exactly like followed replicas (on the core node a core admitted through quarantine counts only after the block 0 its main store received was compared with the gated one), and (c) own writers were pushed to the length the seeder advertises for them (browser: confirmed by `group-pushed { key, length }`); all bounded by the run's deadline. Against a seeder without `varanda/groups/1` (groups unsupported) the browser's own group pushes are not once-run goals, so a once run finishes. The goal of a binding diary is met at its present length (section 2 "Clearing deleted bytes"), so a deleted post no longer keeps a once run open, and join blocks after a cleared hole are read. Group writer and root cores keep their own tracking, since no delete records exist for them. Goals G1 to G3 of section 6 are unchanged. The browser's write gate for a group write confirms the own writer's length, not only the diary. For each community the goals are C's root, every page of its CWs, and the binding diaries, as for groups. +- **Completion goals** (`--once` and the browser's once runs). Each group goal runs once per run and is met when (a) every writer page of every joined group was fetched (not `incomplete`) or the channel timed out, (b) every admitted foreign writer, and the diary that binds it up to its join, reached the length its seeder advertised, exactly like followed replicas (on the core node a core admitted through quarantine counts only after the block 0 its main store received was compared with the gated one), and (c) own writers were pushed to the length the seeder advertises for them (browser: confirmed by `group-pushed { key, length }`); all bounded by the run's deadline. A writer a seeder keeps sparse ("Sparse copies") never reaches its signed length contiguously, so (b) also holds for a foreign writer held up to the advertised length and past it with a hole once every connected peer that synced has answered for the blocks below the signed length (up to `MAX_LOG_BLOCKS`) and every block one of them advertises there is held (`advertisedHeld` in `sparse-held.js`, checked again every `SPARSE_CHECK_MS` = 100 ms while the goal waits; in the browser, which may run that goal for as long as a tab is open, the wait doubles up to 5 s while the copy's held range stays the same, and a download or a peer coming or going brings it back to 100 ms). Past a peer's contiguous length its bitfield arrives only as the answer to a download range's want: one want per bitfield segment (2097152 blocks), answered from the segment's first block up to the last page the peer holds anything in. So a peer counts as having answered once the first page of each segment of that range arrived; a later page it holds nothing in never comes. (c) for an own writer the own diary cut asks the seeder to hold it up to `max(cut, 1)` (`diaryWriterCut` over the diary's slot walk: 0 once released), never past the length the device holds, and past the cut each of its own `group-leave` and `group-key` blocks, as the seeder's replication peer advertises it or as served to that peer on this connection (a peer never announces a block it gets to one that already holds it). The device asks that peer for its bitfield there (`askHeld` in `sparse-held.js`), since a full copy sends no wants of its own; the core node reads the cuts when the run opens its writers, the browser at each scan of its diary. An own writer the device retired by a join (`retiredWriterCut`, "Writers named by a node") is cut at the length it holds, so nothing past its cut waits for the seeder, unless the cut went at a lower mark already on it: its leaves and key blocks past that mark are confirmed the same way. The account service's group fold for a page (`apps/cli/lib/account/pages-groups.js`, section 12) counts a listed writer whole on the same rule, so a group with such a writer is no longer partial (`indisponivel`): its fetch ends, before `FETCH_TIMEOUT_MS`, once every block the seeder advertises below the signed length is held (polled every 50 ms). Roots and diaries keep the contiguous rule. Against a seeder without `varanda/groups/1` (groups unsupported) the browser's own group pushes are not once-run goals, so a once run finishes. The goal of a binding diary is met at its present length (section 2 "Clearing deleted bytes"), so a deleted post no longer keeps a once run open, and join blocks after a cleared hole are read. Group writer and root cores keep their own tracking, since no delete records exist for them. Goals G1 to G3 of section 6 are unchanged. The browser's write gate for a group write confirms the own writer's length, not only the diary. For each community the goals are C's root, every page of its CWs, and the binding diaries, as for groups. - **Parent paging.** For every local group root held the node reads its header's `community`, and for every local community its key. For each community C it pages C on the groups channel (the root and its CWs), gates each CW like a group writer (a valid header with `group` = C and `member: null`, then the binding diary's block 0 and join block), stores the CWs and their binding diaries, and pages only the roots of C's catalogue (the `/index/community/` rows are a hint, the fold verifies) so that `community.info` can name them. Each root is judged by its own block 0 under a role fixed for the run: a key joined as a group must be a `group` root, a key reached as a parent must be a `community` root, and a `v: 1` header is `header-invalid`; the `group-skipped` reason `not-community` (a parent that is not a community root; the node's own verdict, the seeder is not charged) is kept. A run holds at most groups x 512 plus communities x 512 cores. A node folding a group passes `buildGroup` the parent's input (C's root, its CWs and their joins). The core sync emits `community-parent { key, group }` (`key` the community root C, `group` the group G) the first time a held group root names a community this node does not already hold, and `group-parent { key: C, group: G }` once per local group whose held root names C, held or not (the session uses it to fan `groups.changed` out from C's growth to its groups; the CLI prints neither). - **Diary-bound roots (browser).** Besides kv `groupWriters` and kv `groupReplicas`, the browser sync pages every root its own diary binds now as a replica: the newest `group-join` per writer still `on: true`, among the writers the slot walk counts live or frozen per group ("Limits"), the first 64 groups by live join seq (`diaryGroupKeys`, the seeder's live-join rule). The scan keeps the diary's `group-cut` records with its joins, so a writer that took a slot a cut released binds its root, and the same walk picks the newly bound writers it chases. It reads them at run start from the stored diary and again whenever the own diary is caught up with the seeder, so a second device of the same passkey (same diary, fresh storage) rebuilds its communities and groups from the diary. A writer the diary newly binds to a root already adopted is paged at once (that group alone, ahead of the pass in progress or in a pass of its own), and again after waits of 2, 5 and 10 s (each counted from the previous page, so at about 2, 7 and 17 s) while the seeder does not list it yet; the writers of the run's first scan are not chased, the regular pass asks for them. A diary block the read could not get (not downloaded yet, or a read that failed) is kept, read again on every later scan, when it downloads and, after a failed read, on a short timer, and never holds back the joins after it. A once run does not end while that read of the own length is pending: the roots it adopts are paged before the run's goals are met. Such a root's writers are gated like any replica's, a catalogue child the diary joined is not paged root only (one already paged root only in the run is paged again in full), and a community among them is paged in full with its catalogue. Its own header decides its kind: another root's header naming it as its community never makes it a parent key, so it is never refused as `not-community`. A root no join binds now (every device left) is not adopted, a blocked key never is, and a key with no root at the seeder is dropped by the probe rule and not asked for again in that run. Nothing is appended, no writer is created and nothing is listed in kv for an adopted key until its block 0 is stored (the gate then lists it in kv `groupReplicas`). The core node is unchanged: its diary is per device (section 1), so another device never shares it. - **Automatic device enrolment (browser).** After a sync, a device of the same passkey enrols by itself in each diary-bound root it holds and has no kv `groupWriters` entry for, while the fold says the diary is a `member` there and fewer than `MAX_GROUP_WRITERS_PER_DIARY` - 2 (6) writers of the diary are live in its slot walk there (the root slot included): the last two live slots are spent only by an explicit join. A slot comes back only through the diary's own cut (a disconnected device, section 1, or the cut a join writes on a writer it replaces, "Writers named by a node"), while the walk has room to freeze or release the writer; an older seeder never releases one. It writes what `community.join` or `group.join` writes (a new writer naming the diary's latest leave, and the diary's public `group-join`), communities first, then a group only when this device's community writer is admitted in C (an open community or its creator; in an approval community the diary's admission covers the new community writer, "Admits list their writers", and its groups follow on a later run, once the fold shows it admitted). Geral is planned like any group of C with its own checks: a diary that left Geral is not put back, and Geral's last two slots stay reserved too. A kv writer the diary never bound (an interrupted join) is bound again, reusing that writer only when this device derives its key from the entry's nonce; otherwise the root is skipped (`foreign`), and an explicit join writes a new writer instead. Only a writer this device derived is ever reused by a join. A root that is left, banned, requested, blocked, unheld, an orphan, not joined by the own diary (an opened link) or not foldable yet (no pending join) is skipped, and no admit or grant is written automatically (the fold's coverage of a new device by the diary's admission writes nothing). Rules in [rpc.md](rpc.md) 5.63. The core node is unchanged (its diary is per device, section 1). diff --git a/docs/verification/feat-writer-slot-sparse.md b/docs/verification/feat-writer-slot-sparse.md new file mode 100644 index 00000000..83cb21a8 --- /dev/null +++ b/docs/verification/feat-writer-slot-sparse.md @@ -0,0 +1,296 @@ +# Escritores cortados guardados e lidos esparsos (branch feat/writer-slot-sparse) + +Parte da #350, item 8a, passo 8a-3, sobre `feat/writer-slot-walk`. Decisão do dono (não reaberta aqui): o semeador guarda, dentro da cota normal, as saídas e os blocos de chave de um escritor cortado e um toco de cada escritor liberado. + +## O que mudou + +Dobra (`packages/core/lib/group.js`): + +- `bindCores`: um escritor conta até o primeiro seq que nenhuma cópia tem e, depois desse buraco, só as saídas (`group-leave`) e os blocos de chave (`group-key`), e só quando o buraco está no corte ou depois dele (`max(corte, 1)`). Uma cópia esparsa de um escritor cortado dobra como a inteira; uma cópia parcial, cortada ou não, dobra como o prefixo de sempre. +- Cada cópia traz `signed` (o comprimento que o escritor assinou). Um bloco de chave cujo `auth` cita um escritor de comunidade (CW) cortado vale quando `auth.seq` não passa desse comprimento (`heldSeq = max(lastSeq, held, signed - 1)`), então um CW aparado no corte lê o histórico que a cópia inteira lê. + +Leitores: + +- Nó core: `readHeldBlocks` em `packages/core/lib/store.js` (o prefixo contíguo e, depois do primeiro buraco, só as saídas e os blocos de chave guardados, até `MAX_LOG_BLOCKS`); `readCoreParts` em `member.js` lê os escritores assim e passa `signed: core.length`. As raízes continuam lidas como prefixo. +- Nó do navegador: `readHeldBlocks` em `packages/web-member/lib/node-store.js`; em `member-ops.js`, `writerCoreState` para os escritores, o carimbo da memória da dobra (`groupStamp = replicaStamp`) nomeia as faixas de blocos guardados depois do comprimento contíguo (terceira rodada; antes, cada bloco), `signed` em `foldGroup` e `foldCommunity`, e `foldCommunity` passa a receber as marcas de corte dos diários. +- Serviço de contas: `apps/cli/lib/account/pages-groups.js` lê os escritores listados com `readHeldBlocks` e passa `signed`. + +Semeador (`packages/core/lib/seeder.js`): + +- Escritor cortado (`groupWalkCut` sobre a caminhada): o download para no corte; os blocos depois dele, até `MAX_LOG_BLOCKS`, são buscados de 256 em 256 (`CUT_CHUNK`), lidos só pelo tipo e limpos, menos saídas e blocos de chave (`_cutGroupCore`). O Hypercore guarda os nós da árvore que as provas dos blocos mantidos pedem. +- Liberado (corte 0): toco com o bloco 0, as saídas e os blocos de chave. +- A ligação em `admitted.json` guarda `cut`, `clearedFrom` e `clearedTo`, gravada antes de limpar cada pedaço; o próximo serviço relê essa faixa e limpa o que sobrou de uma queda no meio. +- Cota: na admissão e na partida, só os blocos abaixo do corte; com o núcleo inteiro acima da cota, os blocos abaixo do corte mais os mantidos depois dele, e os bytes deles. Passou: sai com `quota` e a entrada fica recusada, como na admissão (senão cada varredura buscaria de novo). +- Corte menor aplica na hora (também com o núcleo ainda abrindo); maior ou nenhum, só depois que a varredura alcançou o diário, e então o que a limpeza tirou é buscado de novo. Um corte restaurado na partida não limpa nada novo antes de a varredura confirmá-lo. +- Uma entrada recusada antes (`quota`, `header-invalid`) é tentada de novo uma vez a cada corte menor. +- Escritor sem caixa num grupo privado: `header-invalid` enquanto contado, toco depois de liberado (`groupVerdict(..., released)`, na admissão, na partida, no portão e na reverificação de caixa). Se a liberação some, é julgado de novo e sai. + +Metas de conclusão com cópia esparsa (segunda rodada, pontos (a) e (b)): + +- `packages/core/lib/sparse-held.js` (novo, roda no Node, no Bare e no navegador): `advertisedHeld(core, end)` é verdadeiro quando algum par conectado já sincronizou, todo par sincronizado já respondeu pelos blocos abaixo de `end` e todo bloco abaixo de `end` que um deles anuncia está guardado. Depois do comprimento contíguo de um par, o bitfield dele só chega como resposta ao pedido de um intervalo de download, então o par conta como respondido quando chegou a primeira página de cada segmento do bitfield que o intervalo toca (terceira rodada; antes, toda página até o fim). +- `diaryWriterCut(entries, G, W)` em `group.js`: o corte que a caminhada do diário dá ao escritor (0 quando liberado, `null` sem corte), igual ao `groupWalkCut` do semeador. +- Nó core (`sync.js`): na meta de `--once`, um escritor alheio guardado até o anunciado e com buraco depois conta como completo quando `advertisedHeld` vale até `min(comprimento assinado, MAX_LOG_BLOCKS)` (`sparseMet`, conferido de novo a cada `SPARSE_CHECK_MS` = 100 ms enquanto a meta espera). O próprio escritor cortado pelo diário conta como enviado quando o semeador o guarda até `max(corte, 1)`, nunca além do comprimento local, e (terceira rodada) já tem as saídas e os blocos de chave dele depois do corte; os cortes são lidos do diário quando a sincronização abre os escritores. +- Nó do navegador (`web-member/lib/sync.js`): as mesmas duas regras (`sparseMet` em `unmetGroupGoals`; `pushTarget` em `checkMedia`), com os cortes relidos a cada varredura do diário (`fold.cutEntries`, `ownCutEntries`), e `checkMedia` roda de novo depois de cada varredura. +- Serviço de contas (`pages-groups.js`): o `fetch` de um escritor listado termina, antes de `FETCH_TIMEOUT_MS`, quando todo bloco que o semeador anuncia abaixo do comprimento assinado está guardado (conferido a cada 50 ms), e então a cópia conta como completa. Raízes e diários continuam com a regra do contíguo. + +Documentos: `docs/protocol.md` (bit `CAP_WRITER_CUTS`, entrada de `buildGroup` e `buildCommunity` com `signed`, "Slot walk" e "Writers named by a node" sobre o sem caixa, regra do `auth.seq` em "Diary cuts on own writers" e em "Community distributors", parágrafo novo "Sparse copies", "Cut writers" do semeador reescrito; na segunda rodada, "Completion goals" com a cópia esparsa, o próprio escritor cortado e o `fetch` do serviço de contas). Na terceira rodada, "Cut writers" com o fim do pedaço pelo anunciado e a faixa incerta, "Completion goals" com a regra do segmento, as saídas próprias depois do corte e a espera do navegador, e "Sparse copies" com o carimbo por faixas. `docs/rpc.md` não muda: nenhum método nem resultado mudou. + +Terceira rodada (revisão em `d487e5e7`, sem P1): + +- Semeador, pedaço depois do corte lido de pares esparsos (P2-1, `_groupPastRange` e `_cutGroupCore` em `seeder.js`): um pedaço termina quando todo bloco que os pares sincronizados anunciam ali está guardado (`advertisedHeld`), não só quando os 256 blocos chegaram. O que ele guarda ali é mantido (e contado na cota) ou limpo como antes, e o pedaço entra na faixa incerta da ligação (`unsureFrom`, `unsureTo`, dentro da faixa já lida), relida pedaço a pedaço em intervalos que dobram de 1 s a 5 minutos (e, desde a quarta rodada, na hora só quando um par que entra anuncia ali um bloco que falta); um pedaço relido inteiro sai dela. Antes, um semeador que só recebia o escritor de pares esparsos (outro semeador, ou depois de uma saída por `quota` que um corte menor levantou) nunca passava do primeiro pedaço, e uma saída em corte + 256 ou depois nunca era buscada. +- Dono de um escritor cortado (P2-2): o nó core (`ownKeptMissing` em `sync.js`) e o navegador (`seederHoldsKept` em `checkMedia`) só dão o escritor por enviado quando, além de o semeador guardá-lo até `max(corte, 1)`, o par de replicação do semeador anuncia cada `group-leave` e `group-key` próprio do corte em diante, ou recebeu esse bloco nesta conexão (evento `upload`: um par nunca anuncia um bloco que recebe a quem já o tem). `askHeld` (`sparse-held.js`) pede a esse par o bitfield ali, porque uma cópia inteira não manda pedidos próprios. Enquanto falta, o nó core responde `:corte ` e o navegador `:push `, sem `group-pushed`. +- `peerAnswered` (P3-3): um pedido pede um segmento inteiro do bitfield (2097152 blocos), e a resposta traz a primeira página do segmento e as páginas seguintes que o par tem. O par conta como respondido quando chegou a primeira página de cada segmento do intervalo; antes era preciso toda página de 32768 bits até o fim, e um par esparso num escritor longo nunca respondia. +- `readHeldBlocks` (P3-6, `store.js` e `node-store.js`): depois do buraco, pula para o próximo bloco guardado pelo bitfield que o próprio `has()` lê (`firstSet`, presente no Hypercore 10 e no 11, conferido em `node_modules`), em vez de um `has()` por seq. Sessão que não seja a padrão (snapshot, batch) continua com o `has()` por bloco. +- Carimbo da memória da dobra (P3-5, `replicaStamp` em `member-ops.js`): nomeia as faixas de blocos guardados depois do prefixo (`heldRuns` em `sparse-held.js`), nulo só com mais de 4096 faixas. Antes ficava nulo com mais de 4096 blocos entre o contíguo e o comprimento, e um grupo com um toco longo ou um escritor cortado cedo perdia a memória de vez. +- `sparseMet` do navegador (P3-4): a espera entre conferências dobra até 5 s (`SPARSE_CHECK_MAX_MS`) enquanto a faixa guardada da cópia não muda; um download ou um par que entra ou sai volta a 100 ms. Antes eram 100 ms enquanto a aba ficasse aberta. +- Comentários (P3-7, `seeder.js`): `released = (cut === 0)` cobre também o escritor ainda contado cortado em 0 depois de gastas as 16 liberações; os dois ficam como toco. + +Quarta rodada (revisão de `d487e5e7..9b2137f8`, sem P1 nem P2): + +- `_groupPastRange` (P3-b): o laço confere `entry.dropped || this._closing` depois de cada `await` e sai com `null`. Antes, uma queda durante o `await advertisedHeld` deixava o laço parado para sempre, com um ouvinte `peer-add` num núcleo fechado: o `range.destroy()` do Hypercore não rejeita o `done()`, e o `stopQuota` só acorda uma espera já armada. +- Releitura da faixa incerta quando um par entra (P3-a, `_rereadIfOffered` em `seeder.js`, `peerOffers` em `sparse-held.js`): só relê na hora quando o par que entrou, depois de sincronizar e responder ao pedido do bitfield dessa faixa (`askHeld`), anuncia ali um bloco que o semeador não tem. Conferido a cada 1 s desde a entrada, até 10 vezes. Qualquer outro par (um leitor que conecta, um par que reconecta em laço) deixa o recuo como está. Antes, cada `peer-add` voltava o recuo a 1 s e relia a faixa inteira, então o recuo de 5 minutos nunca valia para um escritor cujo aparelho sumiu. +- `peerHolds` ganhou um teste de unidade com blocos servidos (a mutação R4 da terceira rodada passava). + +Quinta rodada: merge da `main` em `4e901c15` (#717, #721, #722), commit `955ae568`, com a mensagem padrão do git. + +- Conflitos, resolvidos mantendo os dois lados: + - `seeder.js`: a varredura dobra o `group-cut` pelo `foldGroupRecord` da #717, dentro de `foldGroupCut`, que guarda a regra deste PR: um corte menor tenta de novo a entrada recusada. + - `sync.js` do nó core: os imports das duas. + - `sync.js` do navegador: + - os imports, o campo `deletedHoles` da #717 e as exportações das duas; + - `fold.cutEntries` saiu: a #722 já guarda os cortes em `fold.joinEntries` (`WALK_TYPES`), e `ownCutEntries` passa a ser essas entradas. + - `member-ops.js`: um comentário. + - `seeder-groups-cuts.test.js`: os imports e o teste novo da #721 antes do de liberação. + - `protocol.md`: o texto da `main` com as frases deste PR. O escritor sem caixa substituído é recusado pela sincronização, e o semeador o guarda como toco. `buildCommunity` recebe `signed?`. +- Além de escolher lados: + - `_pageArrived` (#717): um `group-cut` apagado, buscado uma vez depois da varredura, também passa por `foldGroupCut`. Um corte menor que chega assim levanta a recusa da entrada como na varredura. Há teste novo em `seeder-delete.test.js`, cujo fixture ganhou `groupRefused`. + - Teste de unidade da #721 "a stored binding the scan no longer lists gets its on join too": montava a entrada do diário sem `core` nem `scanned`. O `_reconcileGroups` deste PR chama `_groupCutScan`, que lê os dois, e o arquivo parava com `TypeError` no commit do merge. O fixture ganhou um diário lido até o fim e uma entrada servida. + - Teste "a cut that is gone after a restart fetches again what clearing removed": apagava o registro de corte para o corte sumir. Com a #717 um `group-cut` apagado continua valendo, então o semeador nunca limpava o bloco e o teste esgotava o tempo em "corte apagado no semeador". Virou dois testes: + - "a cut its diary deleted still applies at the seeder, after a restart too": o bloco e o corte ficam, também depois de reiniciar, e nada limpo é buscado de novo. + - "a persisted cut the diary does not give is dropped once the scan caught up, and what clearing removed is fetched again": o mesmo caminho de antes (`caught && cut !== now`), agora com um corte guardado no `admitted.json` que o diário não tem e W limpo depois dele com o semeador parado. +- Interações conferidas: + 1. **Teto de 32 escritores por diário e grupo (#722) e guarda esparsa.** O leitor conta núcleos guardados pelo seq da entrada provada (`othersHolding`, `MAX_GROUP_HELD_PER_DIARY`). O semeador conta ligações por diário e grupo, com o mesmo 8 + 8 + 16. Nenhum dos dois olha os blocos, então um escritor cortado guardado esparso ocupa uma vaga como o inteiro. Quem conta, nos dois, é a caminhada do diário. A guarda esparsa só muda quais blocos ficam, e a dobra (`bindCores`) dobra a cópia esparsa como a inteira. + 2. **Cortes de substituição automáticos (#722).** `retiredWriterCut` põe o corte um depois do último seq que o aparelho tem, isto é, no comprimento do escritor; o teste 7 de `community-member-leave-5.test.js` confere isso. + - A meta do dono vira `min(comprimento, max(corte, 1))`, que é o comprimento. No nó core `ownKeptMissing` nem é chamado, e no navegador `seederHoldsKept` devolve `true` (`cut >= length`). + - Só quando o corte vai na marca menor que já estava no escritor sobram saídas e chaves depois dela. Elas são confirmadas como as de qualquer corte, pelo caminho que Q1, Q2, R1 e R3 seguram. + - O escritor aposentado também deixa de ser o `writer` do grupo em `groups.json`, então nem entra na meta. + 3. **O seq da listagem (#721, `onSeq`) é um seq do diário.** Ele vem da caminhada que a varredura do diário monta. A guarda esparsa só vale para núcleos de escritor (raízes e diários seguem pelo contíguo), então não o muda. O teste novo da #721 passa com a guarda esparsa. + 4. **Busca única dos blocos apagados mantidos (#717).** Acontece nos diários (`entry.deleted`, `groupSeqs`, `fold.deletedHoles`). `keptPast`, `keptPastBytes` e a cota do que fica são dos núcleos de escritor, que não têm registros de apagar. O único encontro é um `group-cut` apagado que chega depois: ele baixa o corte pelo caminho da varredura (`_reconcileGroups`, `_groupCutScan`, `_applyGroupCut`, que tira de `keptPast` o que fica abaixo do novo `clearedFrom`), e agora também levanta a recusa. + +## Como verificar + +Cada arquivo sozinho, a partir da raiz, com `node `, e `npx standard` nos arquivos mudados: + +| Arquivo | Testes | Asserções | +|---|---|---| +| `packages/core/test/seeder-groups-cuts.test.js` (testes 7 a 14 novos) | 14/14 | 327/327 | +| `packages/core/test/sync-groups-sparse.test.js` (novo, DHT local) | 1/1 | 12/12 | +| `packages/core/test/group-private.test.js` | 128/128 | 898/898 | +| `packages/core/test/group-member-6.test.js` | 10/10 | 72/72 | +| `packages/web-member/test/group-memo.test.js` | 11/11 | 60/60 | +| `packages/web-member/test/group-ops.test.js` | 57/57 | 436/436 | +| `packages/core/test/seeder-private.test.js` | 4/4 | 34/34 | +| `packages/core/test/seeder-groups.test.js` | 43/43 | 262/262 | +| `packages/core/test/seeder-groups-3.test.js` | 6/6 | 33/33 | +| `packages/core/test/seeder-groups-waits.test.js` | 12/12 | 61/61 | +| `packages/core/test/seeder-communities.test.js` | 9/9 | 59/59 | +| `apps/cli/test/pages-groups.test.js` | 3/3 | 153/153 | +| `apps/cli/test/account-pages-groups.test.js` | 1/1 | 49/49 | + +Segunda rodada (depois do merge da `main` com #710, #711, #713 e #714), cada arquivo sozinho: + +| Arquivo | Testes | Asserções | +|---|---|---| +| `packages/core/test/sync-groups-sparse.test.js` (asserções novas do `--once`) | 1/1 | 14/14 | +| `packages/web-member/test/sync-groups-sparse.test.js` (novo, DHT local) | 1/1 | 11/11 | +| `apps/cli/test/pages-groups-sparse.test.js` (novo, DHT local) | 1/1 | 8/8 | +| `packages/core/test/sync-groups-3.test.js` | 16/16 | 71/71 | +| `packages/core/test/sync-private.test.js` | 4/4 | 16/16 | +| `packages/core/test/group-member.test.js` | 13/13 | 151/151 | +| `packages/core/test/seeder-groups-cuts.test.js` | 14/14 | 327/327 | +| `packages/web-member/test/sync-groups-2.test.js` | 26/26 | 130/130 | +| `packages/web-member/test/private-ops.test.js` | 24/24 | 231/231 | +| `packages/web-member/test/group-memo.test.js` | 11/11 | 60/60 | +| `apps/cli/test/pages-groups.test.js` | 3/3 | 152/152 (153 na primeira rodada: a asserção sobre a Geral só existe quando a entrada na comunidade também entrou nela, `joined.group`; dá 152 também com o `fetch` novo tirado) | +| `apps/cli/test/account-pages-groups.test.js` | 1/1 | 49/49 | + +`npm run lint` passa. + +O teste de ponta a ponta: A cria a comunidade e dois grupos; B escreve nos dois, congela o escritor de G1 no comprimento (dois posts e a saída depois do corte) e libera o de G2 (um post e a saída). Depois que o semeador guarda os dois esparsos, R, novo, sincroniza só com ele: guarda W1 abaixo do corte mais a saída e W2 como toco, e a comunidade e os dois grupos (info, membros e leitura) dobram igual a quando R recebe as cópias inteiras de B. + +Na segunda rodada ele confere também que o `sync --once` do dono (B) e o do leitor novo (R) terminam `ok` antes do prazo de 30 s. `packages/web-member/test/sync-groups-sparse.test.js` (novo, DHT local e semeador atrás de WebSocket) faz o mesmo no nó do navegador com um escritor congelado: o `--once` de B e o de R terminam `ok`, R guarda o escritor abaixo do corte mais a saída, a saída conta e o post depois do corte não. A saída ali é escrita direto no escritor, como faria um aparelho offline: `group.leave` também desliga a entrada no diário, e esse caso esbarra num problema anterior a este PR (ponto (g) abaixo). `apps/cli/test/pages-groups-sparse.test.js` (novo, arquivo próprio porque `pages-groups.test.js` já usa boa parte do tempo por arquivo do executor): com o semeador guardando esparso o escritor congelado de B, a Página entra na comunidade, entra no grupo e posta, e a leitura dela vem com `incomplete: false`, o post abaixo do corte e sem o de depois. Selar (`group-seal`, grupo privado) passa pela mesma conferência `loaded.partial` e não tem teste próprio aqui. + +Terceira rodada, cada arquivo sozinho: + +| Arquivo | Testes | Asserções | +|---|---|---| +| `packages/core/test/sparse-held.test.js` (novo) | 2/2 | 8/8 | +| `packages/core/test/sync-groups-sparse.test.js` (teste 2 novo) | 2/2 | 20/20 | +| `packages/core/test/seeder-groups-cuts.test.js` (teste 15 novo) | 15/15 | 336/336 | +| `packages/core/test/sync-groups-3.test.js` | 16/16 | 71/71 | +| `packages/core/test/group-member.test.js` | 13/13 | 151/151 | +| `packages/core/test/group-member-6.test.js` | 10/10 | 72/72 | +| `packages/core/test/sync-private.test.js` | 4/4 | 16/16 | +| `packages/web-member/test/sync-groups-sparse.test.js` (testes 1 e 3 novos) | 3/3 | 27/27 | +| `packages/web-member/test/sync-groups-2.test.js` | 26/26 | 130/130 | +| `packages/web-member/test/private-ops.test.js` | 24/24 | 231/231 | +| `packages/web-member/test/group-memo.test.js` (teste 2 novo) | 12/12 | 65/65 | +| `apps/cli/test/pages-groups-sparse.test.js` | 1/1 | 8/8 | +| `apps/cli/test/pages-groups.test.js` | 3/3 | 153/153 (a asserção sobre a Geral voltou a rodar, `joined.group`, como na primeira rodada) | + +`npm run lint` passa. + +O teste 15 de `seeder-groups-cuts.test.js` monta o caso P2-1: o primeiro semeador guarda W esparso (corte 3; cinco posts, mais 300, a saída e mais um post, então a saída fica depois de corte + 256); B sai, e um segundo semeador recebe W só do primeiro. Ele guarda a saída, conta-a na cota (`keptPast`) e lista como incerta a faixa lida só de pares esparsos (`unsureFrom` 3, `unsureTo` o comprimento); quando B volta com W inteiro, relê a faixa, limpa de novo o que não fica e tira a faixa incerta da ligação. O teste 2 de `packages/core/test/sync-groups-sparse.test.js` e o teste 3 de `packages/web-member/test/sync-groups-sparse.test.js` seguram `_groupPastRange` do semeador: com a saída só no aparelho, o `--once` do dono não termina `ok` e diz o que falta (nó core `:corte `, navegador `:push ` e nenhum `group-pushed`); solto o semeador, termina `ok`. O caso do `peerAnswered` (`packages/core/test/sparse-held.test.js`, teste 1) usa um escritor de 40000 blocos e um par com `[0, 10)` mais o bloco 500, como o repro da revisão. O pulo pelo bitfield do `readHeldBlocks` tem teste só no nó core (teste 2 do mesmo arquivo); o do navegador são as mesmas quatro linhas, sem teste próprio. O teste da espera do `sparseMet` (P3-4) é de unidade, sobre `sparseCheckDelay`; o empurrão por download ou par não tem teste que falhe sem ele, e os testes esparsos do navegador passam com ele. + +Quarta rodada, cada arquivo sozinho com `VARANDA_TEST_CONCURRENCY=1 node `: + +| Arquivo | Testes | Asserções | +|---|---|---| +| `packages/core/test/seeder-groups-cuts.test.js` (teste 15 com o par que nada oferece, teste 16 novo) | 16/16 | 340/340 | +| `packages/core/test/sparse-held.test.js` (testes 3 e 4 novos) | 4/4 | 18/18 | + +`npm run lint` passa. + +No teste 15, antes de B voltar, o recuo do segundo semeador é empurrado para 5 minutos e A abre W vazio: o par novo não oferece nada na faixa incerta, nenhuma passada começa em 3 s e o recuo fica. B volta com W inteiro e a faixa é relida na hora, pelo anúncio de B, não pelo recuo. O teste 16 (novo, de unidade) chama `_groupPastRange` com um núcleo falso cujo `has()` derruba a entrada e leva os pares durante o `advertisedHeld`: a espera tem de terminar com `null` e sem ouvinte no núcleo. Os testes 3 e 4 de `sparse-held.test.js` (novos, de unidade) cobrem `peerHolds` com blocos servidos e `peerOffers`. + +Quinta rodada, depois do merge da `main` em `4e901c15`, cada arquivo sozinho com `VARANDA_TEST_CONCURRENCY=1 node `. Primeiro no commit do merge (`955ae568`), depois de novo nos arquivos que os ajustes mexem: + +| Arquivo | Testes | Asserções | +|---|---|---| +| `packages/core/test/sparse-held.test.js` | 4/4 | 18/18 | +| `packages/core/test/sync-groups-sparse.test.js` | 2/2 | 20/20 | +| `packages/core/test/seeder-groups-cuts.test.js` (no merge: `TypeError` no teste da #721; depois do ajuste do fixture e dos dois testes de corte apagado) | 20/20 | 361/361 | +| `packages/core/test/sync-groups-3.test.js` (rodado de novo depois do corte tardio) | 26/26 | 143/143 | +| `packages/core/test/group-member.test.js` | 15/15 | 158/158 | +| `packages/core/test/group-member-6.test.js` | 10/10 | 72/72 | +| `packages/core/test/sync-private.test.js` | 4/4 | 16/16 | +| `packages/core/test/seeder-delete.test.js` (35/35, 288/288 no merge; com o teste do corte tardio) | 36/36 | 290/290 | +| `packages/core/test/sync-groups-7.test.js` | 1/1 | 12/12 | +| `packages/core/test/writer-slot-member.test.js` | 9/9 | 61/61 | +| `packages/core/test/sync-groups-2.test.js` | 27/27 | 96/96 | +| `packages/core/test/group-rekey.test.js` | 16/16 | 118/118 | +| `packages/core/test/community-member-leave-5.test.js` | 10/10 | 86/86 | +| `packages/core/test/deletion.test.js` (rodado de novo depois do corte tardio) | 65/65 | 963/963 | +| `packages/core/test/community-fold.test.js` | 121/121 | 1111/1111 | +| `packages/web-member/test/sync-groups-sparse.test.js` | 3/3 | 27/27 | +| `packages/web-member/test/sync-groups-2.test.js` | 29/29 | 139/139 | +| `packages/web-member/test/private-ops.test.js` | 24/24 | 231/231 | +| `packages/web-member/test/group-memo.test.js` | 12/12 | 65/65 | +| `packages/web-member/test/sync-groups-3.test.js` | 1/1 | 14/14 | +| `packages/web-member/test/group-ops.test.js` | 64/64 | 487/487 | +| `packages/web-member/test/community-ops.test.js` | 77/77 | 564/564 | +| `apps/cli/test/pages-groups-sparse.test.js` | 1/1 | 8/8 | +| `apps/cli/test/pages-groups.test.js` | 4/4 | 157/157 | + +`npm run lint` passa. + +Não rodados aqui, por regra do brief: a suíte inteira e a demo (a sessão que abriu esta roda a CI de entrega). + +## Prova de regressão + +Cada correção foi tirada e o arquivo rodado de novo. O brittle para no primeiro erro lançado, então nos casos de "tempo esgotado" o resto do arquivo não roda. + +Dobra (`group.js`, `packages/core/test/group-private.test.js`), com o `bindCores` de `feat/writer-slot-walk`: 126/128 testes, 894/898 asserções. Falham "the signed length keeps the history readable" e "the sparse CW folds like the whole one" (CW aparado no corte e bloco de chave que o cita), "a hole below the cut: the prefix" e "an uncut writer: the prefix". + +Nó core (`packages/core/test/group-member-6.test.js`), com `readBlocks` no lugar de `readHeldBlocks` em `readCoreParts`: 9/10, 71/72, falha "the sparse copy folds like the whole one". + +Nó do navegador (`packages/web-member/test/group-memo.test.js`): + +- `groupCoreState` em vez de `writerCoreState` para os escritores: 9/11, 56/60, falham "the leave past the hole counts" e "the sparse copy folded like the whole one" (grupo) e "the leave past the cut counts" e "the sparse copy folded like the whole one" (comunidade). +- `foldCommunity` sem as marcas de corte: 10/11, 58/60, falham "the leave past the cut counts" e "the sparse copy folded like the whole one" (comunidade). +- Carimbo antigo (`contiguous`, `length`, `fork`): a conferência da memória lança "stored fold differs from a fresh one". + +Semeador (`packages/core/test/seeder-groups-cuts.test.js`), uma mutação por vez: + +| Mutação | Resultado | +|---|---| +| M1 `_cutGroupCore` retorna na hora (nada é limpo) | tempo esgotado "W cortado em 3" (teste 7) | +| M2 cota da admissão sobre o núcleo inteiro | tempo esgotado "group-admitted" (teste 12) | +| M3 bytes mantidos depois do corte fora da cota | tempo esgotado "group-rejected ... quota" (teste 12) | +| M4 sem `caught && cut !== now` (corte que some não é aplicado) | tempo esgotado "W buscado de novo" (teste 11) | +| M5 sem `&& !released` no veredito do sem caixa | tempo esgotado "group-admitted" (teste 14) | +| M6 recusa não levantada por um corte menor | tempo esgotado "group-admitted" (teste 14) | +| M7 sem `groupRefused.set` ao sair pela cota do que guarda | 13/14, 326/327, falha "a writer dropped for what it keeps is not fetched again" | +| M8 `_groupCutScan` depois do teste de servido | o teste 13 estoura os 150000 ms | +| M9 veredito da partida sem a liberação | 13/14, 325/327, falham "N served at ready" e "not purged" | +| M10 `cut`, `clearedFrom` e `clearedTo` fora da ligação | falha "from its persisted cut" (teste 10), depois tempo esgotado "sem corte persistido" (teste 11) | + +Ponta a ponta (`packages/core/test/sync-groups-sparse.test.js`): + +- E1, `readBlocks` em `readCoreParts` (nó core lê os escritores como prefixo): 0/1, 11/12, falha "the sparse copies fold like the whole ones". O semeador e as cópias de R estão certos; só a dobra difere. +- E2, M1 no semeador (escritores cortados guardados inteiros): tempo esgotado "run condition", o semeador nunca fica esparso. + +Segunda rodada, uma mutação por vez (`node ` com a mudança tirada, depois restaurada): + +| Mutação | Arquivo | Resultado | +|---|---|---| +| A1 meta do leitor sem `sparseMet` (nó core) | `packages/core/test/sync-groups-sparse.test.js` | 0/1, 13/14, falha "R's run ends ok before its deadline: tempo esgotado: :2/5, :1/4" | +| A2 meta do próprio escritor no comprimento inteiro (nó core) | o mesmo | 0/1, 13/14, falha "B's run ends ok before its deadline: tempo esgotado: :5/2, :4/1" | +| A3 meta do leitor sem `sparseMet` (navegador) | `packages/web-member/test/sync-groups-sparse.test.js` | 0/1, 10/11, falha "R's run ends ok before its deadline: tempo esgotado: :2/4" | +| A4 `pushTarget` devolve o comprimento inteiro (navegador) | o mesmo | 0/1, 10/11, falha "B's run ends ok before its deadline: tempo esgotado: :push 4" | +| B1 o `fetch` do escritor listado como antes | `apps/cli/test/pages-groups-sparse.test.js` | tempo esgotado "group join" (60 s de `indisponivel`) | +| B2 o `fetch` esparso sem `advertisedHeld` | o mesmo | tempo esgotado "group join" | + +Terceira rodada, uma mutação por vez: + +| Mutação | Arquivo | Resultado | +|---|---|---| +| C1 `peerAnswered` exige toda página de 32768 bits até o fim | `packages/core/test/sparse-held.test.js` | 1/2, 7/8, falha "every advertised block is held well before the deadline" | +| F1 `readHeldBlocks` com um `has()` por seq | o mesmo | 1/2, 7/8, falha "has() asked 40000 times, not once per block" | +| P1 pedaço depois do corte termina só com `range.done()` | `packages/core/test/seeder-groups-cuts.test.js` | tempo esgotado "W lido do primeiro semeador" (teste 15) | +| P2 sem a releitura da faixa incerta | o mesmo | tempo esgotado "W relido inteiro" (teste 15) | +| Q1 dono sem conferir as saídas depois do corte (nó core) | `packages/core/test/sync-groups-sparse.test.js` | 1/2, 18/20, falham "B's run does not count W pushed at its cut alone" e "it waits for the leave" | +| Q2 `askHeld` não pede nada (nó core) | o mesmo | 1/2, 19/20, falha no teste 1 "B's run ends ok before its deadline: tempo esgotado: :corte 4, :corte 3" | +| R1 dono sem conferir as saídas (navegador) | `packages/web-member/test/sync-groups-sparse.test.js` | 1/2, 17/20, falham "B's run does not confirm W at its cut alone", "it waits for W: []" e "and never reports it pushed" | +| R2 `askHeld` não pede nada (navegador) | o mesmo | passa, 2/2, 20/20 (abaixo) | +| R3 blocos servidos não contam (navegador) | o mesmo | 1/2, 18/20, falham "once the seeder reads past the cut the run ends ok: tempo esgotado: :push 4" e "W reported pushed at its length" | +| R4 blocos servidos não contam (nó core) | `packages/core/test/sync-groups-sparse.test.js` | passa, 2/2, 20/20 (abaixo) | +| S1 carimbo com o limite antigo de 4096 blocos | `packages/web-member/test/group-memo.test.js` | 11/12, 63/65, falham "signed with 5990 blocks missing past its prefix" e "a block arriving past the hole changes it" | +| S2 espera sempre de 100 ms | `packages/web-member/test/sync-groups-sparse.test.js` | 2/3, 25/27, falham "twice the last wait while unchanged" e "up to the bound" | + +As linhas R e Q rodaram antes do teste de unidade da S2, por isso 2 testes no arquivo do navegador. R2 passa porque, neste teste, o pedido que `seederHoldsMedia` já faz ao conectar (o segmento 0, antes de chegar a faixa contígua) traz o bitfield depois do corte; `askHeld` cobre o caso em que isso não acontece, e Q2 mostra no nó core que sem ele o dono não termina. R4 passa porque no nó core a resposta do bitfield costuma chegar depois de o semeador buscar a saída; R3 mostra no navegador o caso em que só a contagem do que foi servido confirma, e na quarta rodada o teste de unidade de `peerHolds` (T3, abaixo) falha sem ela. + +Quarta rodada, uma mutação por vez: + +| Mutação | Arquivo | Resultado | +|---|---|---| +| T1 sem conferir a queda depois do `advertisedHeld` | `packages/core/test/seeder-groups-cuts.test.js` | 15/16, 338/340, falham "the wait ends with nothing read" e "and listens on the core no more" (teste 16) | +| T2 cada `peer-add` volta o recuo a 1 s e relê (o código antigo) | o mesmo | 15/16, 338/340, falham "a peer offering nothing there starts no pass" e "and leaves the backoff" (teste 15) | +| T4 um par que oferece um bloco não relê nada | o mesmo | tempo esgotado "W relido inteiro" (teste 15: com o recuo em 5 minutos, só o anúncio de B relê) | +| T3 `peerHolds` sem os blocos servidos (R4) | `packages/core/test/sparse-held.test.js` | 3/4, 17/18, falha "a block served to it counts" | + +Quinta rodada, depois do merge, uma mutação por vez: + +| Mutação | Arquivo | Resultado | +|---|---|---| +| U1 `_pageArrived` dobra o corte apagado tardio só com `foldGroupRecord` (o código da #717) | `packages/core/test/seeder-delete.test.js` | 35/36, 289/290, falha "the writer the late cut released is tried again" (teste 32) | +| V1 sem `caught && cut !== now` (a M4 de antes) | `packages/core/test/seeder-groups-cuts.test.js` | tempo esgotado "W buscado de novo" (teste 15) | + +A M4 rodou sobre o teste que apagava o registro de corte, que a #717 tornou impossível (acima); a V1 é a mesma mutação sobre o teste que o substitui. No commit do merge, antes do ajuste do fixture, `seeder-groups-cuts.test.js` parava com `TypeError` em `scanCaughtUp` no teste da #721: é o encontro do `_groupCutScan` deste PR com o `_reconcileGroups` da #721, não uma mutação. + +## Decisões tomadas aqui + +- A regra do buraco mora na dobra (`bindCores`), não só nos leitores: assim o nó core, o nó do navegador e o serviço de contas dobram igual mesmo com cópias diferentes do mesmo escritor, e um bloco depois de um buraco abaixo do corte nunca conta. +- Os escritores são lidos com `readHeldBlocks`; as raízes continuam com o prefixo (nunca são cortadas por um diário). +- No nó do navegador, `foldCommunity` passa a receber as marcas de corte: sem elas, um CW cortado guardado esparso dobrava diferente da cópia inteira. +- O semeador grava `cut`, `clearedFrom` e `clearedTo` antes de limpar; uma recontagem a cada serviço limpa o que uma queda deixou. A faixa lida fica numa peça só: um corte menor lê para baixo a partir de `clearedFrom` antes de seguir para cima. +- Um corte restaurado do `admitted.json` serve na hora, mas não limpa nada novo antes de a varredura do diário confirmá-lo (`cutConfirmed`). Corte menor aplica na hora (a varredura só acrescenta marcas, então nunca fica acima do corte do diário); corte maior ou ausente só depois de a varredura alcançar o diário (`scanCaughtUp`). +- Uma entrada recusada é tentada de novo uma vez a cada corte menor que a caminhada dá ao escritor: um corte faz caber na cota quem não cabia, e a liberação faz o sem caixa valer como toco. O custo é no máximo uma busca do bloco 0 por registro de corte. +- Um escritor que sai por causa do que guarda fica com a entrada recusada (`groupRefused`), como uma recusa na admissão; sem isso ele seria admitido, buscado e derrubado a cada varredura. +- A cota do que fica guardado usa `_overQuota` (núcleo inteiro) como atalho: só conta blocos e bytes quando o núcleo inteiro já passa da cota. Os bytes abaixo do corte são lidos um a um só nesse caso. +- Segunda rodada, (a): o anúncio que conta é o de todo par conectado que já sincronizou, não só o do semeador, e um par que ainda não respondeu pelo intervalo segura a meta. O próprio escritor usa o corte do diário (0 vira 1: o bloco 0 sempre fica), nunca o que o semeador lista. +- Segunda rodada, (b): só os escritores listados ganham a regra nova no serviço de contas; raiz e diário não são cortados e continuam pelo contíguo. O `fetch` ainda espera no máximo `FETCH_TIMEOUT_MS`; a diferença é terminar antes quando o anunciado já está guardado. +- (c) As dobras de catálogo (comunidade no índice) não recebem marcas e leem o prefixo, o que continua seguro. Mantido como está. +- (d) As cotas do lado do leitor usam o comprimento assinado, como já era. Mantido como está. +- (e) O semeador baixa os blocos depois do corte, até `MAX_LOG_BLOCKS`, só para ler o tipo; os pedaços de 256 limitam o que fica guardado além do mantido, não o que trafega. Mantido como está. +- O fragmento de changelog de `feat/writer-slot-walk` (que diz que o semeador guarda o escritor inteiro) não foi mexido: é de outro PR; o deste PR diz o comportamento novo. + +## Riscos residuais e pontos em aberto + +- (a) e (b) feitos na segunda rodada (acima). +- (c), (d) e (e): decisões mantidas (acima). Riscos que ficam: (c) uma dobra de catálogo vê menos de um CW cortado do que a dobra completa (só o prefixo), nunca mais; (d) um escritor esparso perto da cota de um leitor conta pelo comprimento assinado, não pelo que o leitor guarda; (e) a banda do semeador para um escritor cortado longo continua a do escritor inteiro até `MAX_LOG_BLOCKS`. +- (f) Não feito: parado para decisão (pedido da rodada). As sincronizações (core e navegador) ainda recusam com `header-invalid` o cabeçalho sem caixa num grupo privado antes de guardá-lo, então um leitor que nunca o teve fica com a dobra `incomplete`, embora o semeador mantenha o toco. Fazer exige que o veredito do portão dependa da caminhada do diário que liga o escritor (o registro de corte 0, não só o bloco de entrada que o I1 admite), levantar a recusa da rodada (`refused`) quando essa caminhada muda, guardar só `[0, 1)`, pular esse escritor em `recheckGroupBoxes`, na revalidação da partida e na meta de conclusão, e decidir o que acontece quando a liberação some (registro de corte apagado). Isso muda as regras de admissão (I1 a I4) nas duas sincronizações. +- (g) Encontrado na segunda rodada, anterior a este PR (vem da #713): o semeador lista um escritor que o diário desligou com `group.leave` com o seq desse `group-join` `on: false`, e o portão do leitor exige `on: true` naquele seq, então recusa como `unbound` (no navegador, visto no teste: `group-skipped` com `reason: 'unbound'`; no nó core pela leitura do código, `joinVerdict`), cobra como dica ruim do semeador e nunca guarda o escritor. Com o diário guardado, a caminhada espera o escritor e a dobra pode ficar `incomplete`. Não mexido aqui. Quinta rodada: a #721, que entrou no merge, lista o escritor que saiu pelo seq do `group-join` `on: true` mais novo (`onSeq`). Conferido só pela leitura do código e pelo teste de unidade da #721, que passa aqui; o teste do navegador com `group.leave` não foi refeito. +- O "respondeu" de `advertisedHeld` é uma aproximação: a primeira página de um segmento criada por outra mensagem do par (o anúncio de um bloco só, um `range` de comprimento 1, por exemplo) antes da resposta ao pedido faz o par contar como respondido cedo. A faixa contígua do par não cria página, então isso só acontece com um bloco anunciado avulso nos primeiros 32768 blocos do segmento. O efeito é um `--once` terminar antes de um bloco anunciado chegar: a dobra trata a cópia como o prefixo (o que está depois do buraco e não chegou não conta) até a próxima sincronização, sem erro. +- Um semeador que só tem pares esparsos de um escritor cortado guarda o que eles têm depois do corte; uma saída ou um bloco de chave que nenhum par conectado tem só chega quando um par mais completo aparece (a faixa incerta é relida na hora quando um par que entra anuncia ali um bloco que falta e, sem isso, a cada até 5 minutos). Um par que só tem esse bloco depois da faixa contígua dele conta pela resposta ao pedido do bitfield; se ela não chega em 10 s, vale o recuo. +- No nó core, os cortes do próprio diário são lidos uma vez, quando a sincronização abre os escritores: um corte escrito durante a mesma execução só vale na próxima. diff --git a/packages/core/lib/group.js b/packages/core/lib/group.js index 72d70136..2aca2e43 100644 --- a/packages/core/lib/group.js +++ b/packages/core/lib/group.js @@ -887,6 +887,19 @@ function walkHasRoom (joins, marks, G, extra = []) { } } +// The cut a diary's slot walk for G gives its writer W, as a seeder keeps W +// (seeder.js groupWalkCut): 0 once released, its lowest cut while counted, +// null with none. `entries`: the diary's records as read ({ seq, record }). +// Never throws. +function diaryWriterCut (entries, G, W) { + try { + const w = slotWalk(groupJoinHistory(entries, G), groupCutHistory(entries, G), G) + if (w.released.has(W)) return 0 + if ((w.live.has(W) || w.frozen.has(W)) && w.cut.has(W)) return w.cut.get(W) + } catch {} + return null +} + function slotWalkResult (s, gap = false, bound = false) { const live = new Set() const frozen = new Set() @@ -1344,6 +1357,20 @@ function lastOn (events) { return !!events && events.length > 0 && events[events.length - 1].on } +// The blocks a writer's cut keeps from the cut on. +function keptPastCut (record) { + return record.type === 'group-leave' || record.type === 'group-key' +} + +// How many blocks from 0 on some copy of a core holds, valid or not. +function heldPrefix (copies) { + const held = new Set() + for (const x of copies) for (const seq of x.log.raw.keys()) held.add(seq) + let n = 0 + while (held.has(n)) n++ + return n +} + function pushTo (map, k, v) { const list = map.get(k) if (list) list.push(v) @@ -1766,7 +1793,7 @@ function bindCores (input, asCommunity) { if (!isHex64(key) || key === G) continue const log = readLog(w.entries, false, null, priv, asCommunity) if (log.header === null && !log.raw.has(0)) continue - pushTo(copies, key, { log, diary: w.diary }) + pushTo(copies, key, { log, diary: w.diary, signed: Number.isSafeInteger(w.signed) && w.signed > 0 ? w.signed : 0 }) } } // Every copy with a block 0 takes part in the merge, whatever its header @@ -1826,14 +1853,24 @@ function bindCores (input, asCommunity) { // reads, or seen by a reader whose hole hides that join) still cuts. const held = marks.cuts.get(h.diary) const cut = Math.min(sl.cut, held && held.has(key) ? held.get(key) : Infinity) - const end = Math.max(cut, 1) - const records = cut === Infinity ? log.records : log.records.filter((x) => x.seq < end || x.record.type === 'group-leave' || x.record.type === 'group-key') + const end = cut === Infinity ? Infinity : Math.max(cut, 1) + // Seeders hold a cut writer sparse, and readers read past a hole (#350): + // a copy counts up to its first hole, and past it only the blocks a cut + // keeps, once the hole is at or past the cut. A sparse copy then folds + // like the whole one, and a partial copy, cut or not, as the prefix it + // always was. + const prefix = heldPrefix(list) + const records = log.records.filter((x) => x.seq < Math.min(prefix, end) || (prefix >= end && keptPastCut(x.record))) // A CW's generation counts its header's rejoin only when the diary's // earliest on-join of it names the same leave (dropStaleLeaves). const c = { key, diary: h.diary, member: h.member, rejoin: h.rejoin, at: h.at, ...splitReacts(records) } if (cut !== Infinity) { c.cut = cut c.held = log.records.length ? log.records[log.records.length - 1].seq : 0 + // The length its writer signed, which a sparse copy proves without + // holding the blocks past the cut. + const signed = Math.max(0, ...list.map((x) => x.signed)) + if (signed > 0) c.signed = signed } c.frozen = sl.frozen c.released = sl.released @@ -2447,7 +2484,7 @@ function foldCore ({ G, header, creator, rootBound, rootEntry, now, C = null, bo const a = n.record.auth if (cs === null || a === null || holdsG(n)) return null const d = n.diary - // Judged against the CW's held length, not its length after a diary + // Judged against the CW's signed length, not its length after a diary // cut: a cut never makes the history a key block handed out unreadable // (sending still needs the CW in C's authorityWriters, which a cut CW is not). if (a.writer !== cwOf.get(n.writer) || cs.cwDiary.get(a.writer) !== d || !cs.lastSeq.has(a.writer) || a.seq > cs.heldSeq.get(a.writer) + 1) return null @@ -3412,8 +3449,10 @@ function foldCore ({ G, header, creator, rootBound, rootEntry, now, C = null, bo cwDiary: writerDiary, counted, lastSeq: state.lastSeq, - // The last seq held per counted CW before any diary cut. - heldSeq: new Map(logs.map((l) => [l.key, Math.max(state.lastSeq.get(l.key) || 0, l.held || 0)])), + // The last seq per counted CW before any diary cut: of a cut CW, the + // last its signed length proves, so a copy trimmed at the cut reads + // the history a whole copy reads (#350). + heldSeq: new Map(logs.map((l) => [l.key, Math.max(state.lastSeq.get(l.key) || 0, l.held || 0, (l.signed || 0) - 1)])), rootAdminC: rootAdmin, rawAdminC: rawAdmin, bansC: bans, @@ -3921,6 +3960,7 @@ module.exports = { groupCutHistory, groupConfirmHistory, slotWalk, + diaryWriterCut, slotWalkState, replacedWriterCut, retiredWriterCut, diff --git a/packages/core/lib/member.js b/packages/core/lib/member.js index d2fc8be0..37ebd665 100644 --- a/packages/core/lib/member.js +++ b/packages/core/lib/member.js @@ -59,6 +59,7 @@ const { readAll, readRaw, readBlocks, + readHeldBlocks, appendRecord, appendRecords, readConfig, @@ -1352,7 +1353,7 @@ async function readCoreParts (identity, key, { blocked, revokedIssuers }, kind) const h = decodeBlock(await core.get(0, { wait: false, raw: true })) if (validateRecord(h, 0) === null) localDiaries.add(hex(core.key)) if (groups.validateWriterHeader(h) !== null || h.group !== key) continue - logs.push({ key: hex(core.key), diary: h.diary, entries: await readBlocks(core), length: core.contiguousLength, signed: core.length }) + logs.push({ key: hex(core.key), diary: h.diary, entries: await readHeldBlocks(core), length: core.contiguousLength, signed: core.length }) } finally { await core.close() } diff --git a/packages/core/lib/seeder.js b/packages/core/lib/seeder.js index 4be55c22..350da555 100644 --- a/packages/core/lib/seeder.js +++ b/packages/core/lib/seeder.js @@ -90,16 +90,22 @@ // diary that count are the ones its slot walk counts (lib/group.js // slotWalk, over its group-join and group-cut records): at most // MAX_GROUP_WRITERS_PER_DIARY live and MAX_FROZEN_WRITERS_PER_DIARY frozen, -// plus each of the up to MAX_WRITER_RELEASES_PER_DIARY writers it released -// (liveGroupJoins). Listed in admitted.json (`groups: { K: { diary, group, -// seq } }`), persisted before the open, purged before un-listing. +// plus a stub of each of the up to MAX_WRITER_RELEASES_PER_DIARY writers it +// released (liveGroupJoins). Listed in admitted.json (`groups: { K: { diary, +// group, seq, cut, clearedFrom, clearedTo } }`), persisted before the open, +// purged before un-listing. // Leaving (a join turned off) NEVER purges: it only marks the writer as // left in the group index (lib/group-index.js), so the moderation record of // a departed member stays verifiable, and a counted writer whose join is -// off is still admitted. A writer its diary cut (frozen or released) is -// held whole, like any writer: a reader folds from a contiguous copy, so -// nothing past the cut is cleared yet. Purges happen for a dropped or blocked diary, a root its blocked or revoked -// creator bound, a root key in the blocklist snapshot, a fork or the +// off is still admitted. A writer its diary cut keeps its blocks below the +// cut and, past it, only its group-leave and group-key blocks, which the +// fold still counts and readers read past a hole (_cutGroupCore): the rest +// is fetched, read and cleared, and the quota counts what is kept. A writer +// cut at 0 (released, or cut at 0 while counted once the releases are +// spent) is such a stub: block 0, its leaves and its key blocks, also a +// replaced boxless writer of a private group, which readers +// then reject by its header. Purges happen for a dropped or blocked diary, +// a root its blocked or revoked creator bound, a root key in the blocklist snapshot, a fork or the // quota. // Group keys never travel on the admission channel; members learn writer // sets over 'varanda/groups/1' (lib/groups-net.js) and the /index routes. @@ -223,6 +229,7 @@ const { const { validateMediaHeader, latestMediaCores } = require('./media') const { validateGroupHeader, validateWriterHeader, validateGroupRecord, groupJoins, slotWalkState, slotWalkStep } = require('./group') const { createGroupIndex } = require('./group-index') +const { advertisedHeld, peerAnswered, peerOffers, askHeld } = require('./sparse-held') const { attachGroups } = require('./groups-net') const { presentState, presentLength, clearDeleted, downloadExcept, downloadExceptPages, pageTargets, deletedRanges, shownMedia, GROUP_HISTORY_TYPES } = require('./deletion') const { createEngagementIndex } = require('./engagement-index') @@ -655,7 +662,7 @@ function mediaBinding (value) { // creator left for another reason (quota, its own drop). function groupBinding (value) { if (!isPlainObject(value)) return null - const { diary, group, seq, by, bound, issuer } = value + const { diary, group, seq, by, bound, issuer, cut, clearedFrom, clearedTo, unsureFrom, unsureTo } = value if (typeof diary !== 'string' || !HEX64.test(diary)) return null if (typeof group !== 'string' || !HEX64.test(group)) return null if (!Number.isSafeInteger(seq) || seq < 1) return null @@ -663,6 +670,22 @@ function groupBinding (value) { if (typeof by === 'string' && HEX64.test(by)) out.by = by if (bound === true) out.bound = true if (out.bound === true && typeof issuer === 'string' && HEX64.test(issuer)) out.issuer = issuer + // A writer its diary cut is kept below `cut` and, in [clearedFrom, + // clearedTo), only its leaves and key blocks (_cutGroupCore); cut 0 is a + // stub. A range that does not fit the cut reads as nothing cleared yet. + // [unsureFrom, unsureTo) inside it was read from sparse peers only and is + // read again; one that does not fit reads as read whole. + if (Number.isSafeInteger(cut) && cut >= 0) { + const below = keptBelow(cut) + const fits = Number.isSafeInteger(clearedFrom) && Number.isSafeInteger(clearedTo) && below <= clearedFrom && clearedFrom <= clearedTo + out.cut = cut + out.clearedFrom = fits ? clearedFrom : below + out.clearedTo = fits ? clearedTo : below + if (fits && Number.isSafeInteger(unsureFrom) && Number.isSafeInteger(unsureTo) && clearedFrom <= unsureFrom && unsureFrom < unsureTo && unsureTo <= clearedTo) { + out.unsureFrom = unsureFrom + out.unsureTo = unsureTo + } + } return out } @@ -816,8 +839,13 @@ async function localMediaVerdict (core, diaryHex) { // writer's `box` must match it, hex64 in a private group and null in a // public one, else 'header-invalid' as in the fold and member sync. A // version 1 header (the validators' 'legacy') is 'header-invalid' too: no -// wire reason of its own, and the startup re-gate purges it. -function groupVerdict (raw, key, group, diary, visibility = null) { +// wire reason of its own, and the startup re-gate purges it. `released`: the +// writer is kept as a stub, cut at 0 (groupWalkCut: released by its diary, +// or cut at 0 while still counted once the releases are spent), so a +// boxless writer of a private group (one its diary replaced) stays as a +// stub, which readers fetch and reject by its header instead of waiting +// for it (#350). +function groupVerdict (raw, key, group, diary, visibility = null, released = false) { try { const header = decodeBlock(raw) if (header === null) return 'header-invalid' @@ -828,7 +856,7 @@ function groupVerdict (raw, key, group, diary, visibility = null) { if (validateWriterHeader(header) !== null) return 'header-invalid' if (header.group !== group) return 'wrong-group' if (header.diary !== diary) return 'wrong-diary' - if (visibility === 'private' && header.box === null) return 'header-invalid' + if (visibility === 'private' && header.box === null && !released) return 'header-invalid' if (visibility === 'public' && header.box !== null) return 'header-invalid' return null } catch { @@ -841,18 +869,18 @@ function groupVerdict (raw, key, group, diary, visibility = null) { async function localGroupVerdict (core, key, binding, visibility = null) { try { const raw = await core.get(0, { wait: false, raw: true }) - return { reason: groupVerdict(raw, key, binding.group, binding.diary, visibility), raw } + return { reason: groupVerdict(raw, key, binding.group, binding.diary, visibility, key !== binding.group && binding.cut === 0), raw } } catch { return { reason: 'header-invalid', raw: null } } } // Writer cores one diary may hold at a seeder in one group, left ones -// included (leaving never frees a slot): its live and frozen slots, and -// each released writer, held whole (#350). +// included (leaving never frees a slot): its live and frozen slots, and a +// stub of each released writer (#350). const MAX_GROUP_HELD_PER_DIARY = MAX_GROUP_WRITERS_PER_DIARY + MAX_FROZEN_WRITERS_PER_DIARY + MAX_WRITER_RELEASES_PER_DIARY // And across all its groups: its share of the group cores, each released -// writer, held whole, included. It stays at the live and frozen slots of +// writer's stub included. It stays at the live and frozen slots of // every group (owner decision). const MAX_GROUP_CORES_PER_DIARY = MAX_GROUPS_PER_DIARY * (MAX_GROUP_WRITERS_PER_DIARY + MAX_FROZEN_WRITERS_PER_DIARY) // Waits before a group core that timed out or found no peer is tried again @@ -989,6 +1017,94 @@ function foldGroupRecord (entry, i, r) { entry.groupWalks.set(r.group, fresh.get(r.group)) } +// A lower cut keeps less (the quota) and a release keeps a boxless writer +// (groupVerdict): a join refused before is tried again, once per lower cut, +// whether the scan read the cut or fetched it late as a deleted hole. +function foldGroupCut (entry, i, r) { + const before = groupWalkCut(entry.groupWalks, r.group, r.writer) + foldGroupRecord(entry, i, r) + const after = groupWalkCut(entry.groupWalks, r.group, r.writer) + if (after !== null && (before === null || after < before)) entry.groupRefused.delete(r.writer) +} + +// The cut a diary's walk for G gives writer W: 0 once released (or cut at 0 +// with no release left), its lowest cut while counted, null with none. +function groupWalkCut (walks, group, writer) { + const s = walks.get(group) + if (s === undefined || writer === group) return null + const st = s.state.get(writer) + if (st === 'released') return 0 + if ((st === 'live' || st === 'frozen') && s.cut.has(writer)) return s.cut.get(writer) + return null +} + +// The blocks of a writer cut at `cut` a seeder keeps in full: those below +// it, block 0 always. +function keptBelow (cut) { + return Math.max(cut, 1) +} + +// Whether a block past a writer's cut stays at the seeder: a group-leave or +// a group-key, which the fold keeps past a cut (lib/group.js bindCores) and +// readers read past a hole (store.js readHeldBlocks). +function keptPastCut (raw) { + if (raw === null || raw === undefined) return false + try { + const block = decodeBlock(raw) + return block !== null && (block.type === 'group-leave' || block.type === 'group-key') + } catch { + return false + } +} + +// Blocks past a cut read per step of _cutGroupCore: what a seeder stores +// beyond what it keeps stays within one chunk. +const CUT_CHUNK = 256 +// A chunk past a cut whose fetch has not ended is checked against what its +// peers advertise (_groupPastRange): soon, then less often while a peer is +// connected; a peer joining checks it again at once. +const CUT_POLL_MIN_MS = 100 +const CUT_POLL_MAX_MS = 2000 +// A range past a cut that sparse peers only fed (binding unsureFrom, +// unsureTo) is read again (_cutGroupCore) at intervals growing from +// REREAD_MIN_MS to REREAD_MAX_MS, until a pass reads each chunk whole, and +// at once when a peer that joins advertises a block there this seeder +// lacks (_rereadIfOffered): checked every REREAD_PEER_MS from its join, so +// that it has synced and answered, up to REREAD_PEER_TRIES times. +const REREAD_PEER_MS = 1000 +const REREAD_PEER_TRIES = 10 +const REREAD_MIN_MS = 1000 +const REREAD_MAX_MS = 5 * 60 * 1000 + +// Widens range { from, to } (null: none) to cover [from, to). +function widen (range, from, to) { + if (range === null) return { from, to } + return { from: Math.min(range.from, from), to: Math.max(range.to, to) } +} + +// The held blocks of [from, to) a cut keeps, as [seq, bytes], and the runs +// [start, end) of the others, to clear. +async function cutRange (core, from, to) { + const keep = [] + const clear = [] + let start = -1 + for (let i = from; i < to; i++) { + let raw = null + try { + raw = await core.get(i, { wait: false, raw: true }) + } catch {} + if (raw !== null && keptPastCut(raw)) { + keep.push([i, raw.byteLength]) + if (start !== -1) clear.push([start, i]) + start = -1 + } else if (raw !== null && start === -1) { + start = i + } + } + if (start !== -1) clear.push([start, to]) + return { keep, clear } +} + // blocklist.json as stored, unwrapped ({ v: 1, snapshot }: store.js // writeBlocklist), or null. function readSnapshot (dataDir, options) { @@ -1966,7 +2082,7 @@ class Seeder { core = await openMediaReplica(this.store, key) if (core.fork !== 0) reason = 'fork' else if (core.contiguousLength === 0) reason = 'no-header' - else if (this._overQuota(core)) reason = 'quota' + else if (this._groupOverQuota(key === binding.group || binding.cut === undefined ? null : binding.cut, core)) reason = 'quota' else ({ reason, raw } = await localGroupVerdict(core, key, binding, this._groupVisibility(binding.group))) } if (reason === null && (groupSurvivors.length >= this.maxGroupCores || (perGroup.get(binding.group) || 0) >= this.maxGroupWriters)) reason = 'group-full' @@ -2026,7 +2142,7 @@ class Seeder { this._groupIndex.setHeader(key, decodeBlock(raw)) this._keepGroupHeader(key, decodeBlock(raw)) } - this._serveGroup(key, binding, core, { headerOk: true }) + this._serveGroup(key, binding, core, { headerOk: true, restored: true }) if (key === binding.group && !this._served.has(binding.diary)) this._indexKeptRoot(key, false) } // A writer served after its root's header was first recorded in this @@ -2431,6 +2547,15 @@ class Seeder { return length > this.maxLogBlocks || byteLength > this.maxLogBytes } + // A group core's log quota at admission and restore: the whole log's or, + // for a writer cut at `cut`, the blocks it keeps below the cut (its bytes + // and the blocks it keeps past the cut count as they are read, + // _cutGroupCore). + _groupOverQuota (cut, { length, byteLength }) { + if (cut === null) return this._overQuota({ length, byteLength }) + return Math.min(keptBelow(cut), length) > this.maxLogBlocks + } + _serve (key, core, { regate, cert0 = null, pushFrom = core.length }) { // Media scan state (header comment): `scanned` is the next block to read // for media-core records (block 0 is the certificate), `decls` the @@ -3105,7 +3230,7 @@ class Seeder { } if (block.type === 'group-cut') { const r = validRecordCopy(block, i) - if (r !== null) foldGroupRecord(entry, i, r) + if (r !== null) foldGroupCut(entry, i, r) continue } // Folded here, not in _runIndex: that queue pauses at capacity and @@ -3615,7 +3740,8 @@ class Seeder { if (r === null || entry.groupSeqs.has(seq)) return this._noteGroupKept(entry, seq) if (r.type !== 'group-join' && r.type !== 'group-cut') return - foldGroupRecord(entry, seq, r) + if (r.type === 'group-cut') foldGroupCut(entry, seq, r) + else foldGroupRecord(entry, seq, r) entry.groupLive = liveGroupJoins(entry.groupDecls, entry.groupWalks) this._reconcileGroups(key, entry) return @@ -4294,6 +4420,9 @@ class Seeder { if (entry.dropped || !entry.gated || this._closing) return for (const [k, b] of [...this._groups]) { if (b === null || b.diary !== diary) continue + // The cut its walk gives a writer, also while its core still opens + // (_serveGroup reads the binding's). + if (k !== b.group) this._groupCutScan(k, b, entry) if (!this._groupServed.has(k)) continue const live = entry.groupLive.get(k) const listed = live !== undefined && live.group === b.group @@ -4355,6 +4484,81 @@ class Seeder { } } + // A lower cut from the walk applies at once: the scan only adds marks, so + // the diary's cut is never above it. A higher one, or none, only once the + // scan caught up with the diary (a cut restored from admitted.json whose + // block can no longer be read, for example), and what clearing removed is + // fetched again. + _groupCutScan (key, b, entry) { + const cut = groupWalkCut(entry.groupWalks, b.group, key) + const now = Number.isSafeInteger(b.cut) ? b.cut : null + const caught = scanCaughtUp(entry) + if ((cut !== null && (now === null || cut < now)) || (caught && cut !== now)) return this._applyGroupCut(key, b, cut) + const served = this._groupServed.get(key) + if (caught && served && !served.dropped && !served.cutConfirmed) { + served.cutConfirmed = true + if (served.cut !== null) this._cutGroupCore(key, served) + } + } + + // Writer K's cut (null: none) on its binding and, once served, on its + // entry: the download stops at the new cut, a lower cut reads the blocks + // now past it (_cutGroupCore), and a higher one or none fetches again what + // clearing removed below it. + _applyGroupCut (key, b, cut) { + const old = Number.isSafeInteger(b.cut) ? b.cut : null + if (cut === null) { + delete b.cut + delete b.clearedFrom + delete b.clearedTo + delete b.unsureFrom + delete b.unsureTo + } else { + const below = keptBelow(cut) + if (old === null) { + b.clearedFrom = below + b.clearedTo = below + } else if (cut > old) { + b.clearedFrom = Math.max(b.clearedFrom, below) + b.clearedTo = Math.max(b.clearedTo, b.clearedFrom) + } + if (Number.isSafeInteger(b.unsureFrom)) { + b.unsureFrom = Math.max(b.unsureFrom, b.clearedFrom) + if (b.unsureFrom >= b.unsureTo) { + delete b.unsureFrom + delete b.unsureTo + } + } + b.cut = cut + } + this._persist().catch(noop) + const entry = this._groupServed.get(key) + if (!entry || entry.dropped) return + entry.cut = cut + entry.cutConfirmed = true + entry.rereadAt = null + entry.rereadLeft = null + entry.belowBytes = 0 + entry.belowCounted = 0 + for (const [i, bytes] of entry.keptPast) { + if (cut !== null && i >= b.clearedFrom) continue + entry.keptPast.delete(i) + entry.keptPastBytes -= bytes + } + this._rearmGroupDownload(entry) + // A boxless writer kept as a stub is judged again once its cut is no + // longer 0. + if (old === 0 && cut !== 0) { + const visibility = this._groupVisibility(entry.group) + if (visibility !== null) this._recheckGroupBoxes(entry.group, visibility) + } + if (cut === null) { + if (this._overQuota(entry.core)) this._dropGroup(key, 'quota') + return + } + this._cutGroupCore(key, entry) + } + _retryGroups (peer) { if (peer) { if (!peer.socket || peer.seederDropped) return @@ -4565,7 +4769,8 @@ class Seeder { return this._rejectGroup(key, diary, group, 'timeout', seq) } if (!this._groupStillWanted(diary, entry, key, group)) return - const verdict = groupVerdict(res.raw, key, group, key === group ? null : diary, this._groupVisibility(group)) + const cut = key === group ? null : groupWalkCut(entry.groupWalks, group, key) + const verdict = groupVerdict(res.raw, key, group, key === group ? null : diary, this._groupVisibility(group), cut === 0) if (verdict !== null) { entry.groupRefused.set(key, seq) return this._rejectGroup(key, diary, group, verdict, seq) @@ -4575,8 +4780,8 @@ class Seeder { if (key === group) this._keepGroupHeader(group, decodeBlock(res.raw)) // Refusals after a fetch are remembered (this join) or backed off, so a // rescan does not fetch the same core again. A writer its diary cut is - // held whole, so its whole log is judged. - if (this._overQuota(res)) { + // judged on the blocks it keeps below the cut. + if (this._groupOverQuota(cut, res)) { entry.groupRefused.set(key, seq) return this._rejectGroup(key, diary, group, 'quota', seq) } @@ -4618,6 +4823,7 @@ class Seeder { if (wasGone.issuer !== null) rootBinding.issuer = wasGone.issuer } const writerBinding = { diary: owner, group, seq: ownSeq } + if (cut !== null) Object.assign(writerBinding, { cut, clearedFrom: keptBelow(cut), clearedTo: keptBelow(cut) }) this._groups.set(key, key === group ? rootBinding : writerBinding) const ownerServed = this._served.get(owner) if (ownerServed && !ownerServed.dropped) this._markGroupBound(ownerServed) @@ -4652,7 +4858,7 @@ class Seeder { else if (key === group && this._rootsClosed()) late = 'denials-full' else if (!this._groupStillWanted(diary, entry, key, group)) late = 'unbound' else if (core.fork !== 0) late = 'fork' - else if (this._overQuota(core)) late = 'quota' + else if (this._groupOverQuota(key === group || writerBinding.cut === undefined ? null : writerBinding.cut, core)) late = 'quota' if (late === null && this._closing) return if (late !== null) { await core.close().catch(noop) @@ -4690,7 +4896,9 @@ class Seeder { // Serving a group core: live download, fork watcher, the log quota on // every append, no growth broadcast. `expected` and `headerOk` as for // media (_serveMedia): a fresh admission is re-gated once block 0 is local. - _serveGroup (key, { diary, group }, core, { expected = null, headerOk = false, by = diary } = {}) { + _serveGroup (key, { diary, group }, core, { expected = null, headerOk = false, by = diary, restored = false } = {}) { + const binding = this._groups.get(key) + const cut = key !== group && binding && Number.isSafeInteger(binding.cut) ? binding.cut : null const entry = { core, diary, @@ -4709,16 +4917,68 @@ class Seeder { rootScanning: false, rootDecoded: 0, // blocks a root scan decoded (past the group-meta prefilter) rootRetry: null, // the retry of a failed root block read (_retryRootScan) - rootReadFailures: 0 + rootReadFailures: 0, + // A writer its diary cut (binding `cut`): its blocks below the cut and, + // in the range its binding lists as read, only its leaves and key + // blocks (`keptPast`, seq -> bytes; _cutGroupCore). `cutConfirmed` + // once the diary's scan gave the cut: one restored from admitted.json + // clears nothing new before. `belowBytes` the bytes of the blocks below + // the cut counted so far (`belowCounted` of them), `counted` once the + // range read was read again in this serve, `past` the fetch of blocks + // past the cut being read, `cutting`, `cutAgain` and `wake` one pass at + // a time. A read again of the binding's unsure range (`reread` asked, + // `rereadAt` the next chunk, `rereadLeft` the chunks still read from + // sparse peers only in this pass) runs at `rereadTimer`, backing off + // by `rereadDelay`. + cut, + cutConfirmed: !restored, + keptPast: new Map(), + keptPastBytes: 0, + belowBytes: 0, + belowCounted: 0, + counted: false, + past: null, + cutting: false, + cutAgain: false, + wake: null, + reread: false, + rereadAt: null, + rereadLeft: null, + rereadTimer: null, + rereadDelay: REREAD_MIN_MS } const checkQuota = () => { if (entry.dropped || this._closing) return + if (entry.cut !== null) return this._cutGroupCore(key, entry) if (this._overQuota(core)) this._dropGroup(key, 'quota') } + // A block below a cut counts toward the bytes kept, read only while the + // whole log is over the byte quota. + const onDownload = (index) => { + if (entry.cut !== null && index < keptBelow(entry.cut) && core.byteLength > this.maxLogBytes) this._cutGroupCore(key, entry) + } + // A peer that joins may hold what sparse peers did not. + const onPeer = (peer) => { + if (entry.dropped || this._closing) return + const b = this._groups.get(key) + if (b && Number.isSafeInteger(b.unsureFrom)) this._rereadIfOffered(key, entry, peer, REREAD_PEER_TRIES) + } core.on('append', checkQuota) - entry.stopQuota = () => core.off('append', checkQuota) + core.on('download', onDownload) + core.on('peer-add', onPeer) + entry.stopQuota = () => { + core.off('append', checkQuota) + core.off('download', onDownload) + core.off('peer-add', onPeer) + try { + if (entry.past) entry.past.range.destroy() + } catch {} + if (entry.rereadTimer !== null) clearTimeout(entry.rereadTimer) + entry.rereadTimer = null + if (entry.wake !== null) entry.wake() + } entry.stopFork = watchFork(core, (reason) => this._dropGroup(key, reason)) - entry.download = core.download({ start: 0, end: -1 }) + entry.download = core.download(cut === null ? { start: 0, end: -1 } : { start: 0, end: keptBelow(cut) }) entry.stopGrow = watchContiguous(core, () => { this._gateGroup(key, entry) this._scanRoot(key, entry) @@ -4726,6 +4986,7 @@ class Seeder { this._groupServed.set(key, entry) checkQuota() if (entry.dropped) return + this._armReread(key, entry, REREAD_MIN_MS) this._scanRoot(key, entry) // A peer that served block 0 to the quarantine and then withholds it // from the main store must not keep the group's answers incomplete @@ -4742,6 +5003,290 @@ class Seeder { if (!entry.headerOk && core.contiguousLength >= 1) this._gateGroup(key, entry) } + // The download of a served writer for its cut: the blocks below it, block + // 0 always, or the whole log without one. A block a pass cleared under a + // cut that rose meanwhile is fetched again. + _rearmGroupDownload (entry) { + try { + if (entry.download) entry.download.destroy() + if (entry.past) entry.past.range.destroy() + } catch {} + entry.past = null + entry.download = entry.core.download(entry.cut === null ? { start: 0, end: -1 } : { start: 0, end: keptBelow(entry.cut) }) + } + + // Whether a cut writer keeps more than the log quota: the blocks below its + // cut, block 0 always, and the leaves and key blocks kept past it, and + // their bytes. Counted only once the whole log is over the quota. + _groupKeptOver (entry) { + const core = entry.core + if (!this._overQuota(core)) return false + const blocks = Math.min(keptBelow(entry.cut), core.length) + entry.keptPast.size + return blocks > this.maxLogBlocks || entry.belowBytes + entry.keptPastBytes > this.maxLogBytes + } + + // Waits for blocks [from, to) of a cut writer, fetched by one range kept + // across passes: 'done' once all are local; 'sparse' once every block of + // it a connected peer advertises is (sparse-held.js advertisedHeld), as + // when its only sources keep it sparse themselves (another seeder past + // the same cut), whose fetch would never end; null when another pass was + // asked for first or the fetch ended without them. + async _groupPastRange (entry, from, to) { + let p = entry.past + if (p !== null && (p.from !== from || p.to !== to || p.failed)) { + try { + p.range.destroy() + } catch {} + p = null + } + if (p === null) { + const range = entry.core.download({ start: from, end: to }) + const q = { from, to, range, done: false, failed: false, settled: null } + q.settled = range.done().then(() => { q.done = true }, () => { q.failed = true }) + entry.past = q + p = q + } + const core = entry.core + // A drop or a close destroys the range, whose done() then never settles, + // and wakes only a wait already armed: one landing during an await here + // would park this loop for good, listening on a closed core. + const gone = () => entry.dropped || this._closing + let delay = CUT_POLL_MIN_MS + let held = null + while (!p.done && !p.failed && !gone()) { + const all = await advertisedHeld(core, to, from) + if (gone()) break + if (all) { + held = (await core.has(from, to)) ? 'done' : 'sparse' + break + } + if (p.done || p.failed) break + let timer = null + let onPeer = null + const why = await new Promise((resolve) => { + entry.wake = () => resolve('wake') + p.settled.then(() => resolve('settled')) + onPeer = () => resolve('peer') + core.on('peer-add', onPeer) + if (core.peers.length > 0) timer = setTimeout(() => resolve('tick'), delay) + }) + entry.wake = null + core.off('peer-add', onPeer) + if (timer !== null) clearTimeout(timer) + if (why === 'wake') break + delay = why === 'tick' ? Math.min(delay * 2, CUT_POLL_MAX_MS) : CUT_POLL_MIN_MS + } + if (gone()) return null + if (p.done) held = 'done' + if (held === null) return null + // Its blocks no peer advertised would otherwise come later, past the + // read, and stay. + if (!p.done) { + try { + p.range.destroy() + } catch {} + } + if (entry.past === p) entry.past = null + return held + } + + // Reads writer K's unsure range again at once when `peer`, which joined, + // advertises a block there this seeder lacks, checked REREAD_PEER_MS from + // now, once it has synced and answered for the range (asked for its + // bitfield there, as a seeder holding the blocks below the cut sends no + // want past them), up to `tries` times. Any other peer, a reader that + // connects or one reconnecting in a loop, leaves the backoff as it is: a + // writer whose device is gone keeps its unsure range for good. + _rereadIfOffered (key, entry, peer, tries) { + const stale = () => entry.dropped || this._closing || this._groupServed.get(key) !== entry || peer.removed === true + const timer = setTimeout(async () => { + try { + const b = this._groups.get(key) + if (stale() || !b || !Number.isSafeInteger(b.unsureFrom)) return + if (peer.remoteSynced === true) askHeld(peer, [b.unsureFrom]) + if (!peerAnswered(peer, b.unsureTo, b.unsureFrom)) { + if (tries > 1) this._rereadIfOffered(key, entry, peer, tries - 1) + return + } + if (!(await peerOffers(entry.core, peer, b.unsureFrom, b.unsureTo)) || stale()) return + entry.rereadDelay = REREAD_MIN_MS + this._armReread(key, entry, 0) + } catch {} + }, REREAD_PEER_MS) + if (timer.unref) timer.unref() + } + + // Arms the next read again of writer K's unsure range in `delay` ms (none + // without one), replacing any armed before. + _armReread (key, entry, delay) { + if (entry.rereadTimer !== null) clearTimeout(entry.rereadTimer) + entry.rereadTimer = null + const b = this._groups.get(key) + if (entry.dropped || this._closing || !b || !Number.isSafeInteger(b.unsureFrom)) return + entry.rereadTimer = setTimeout(() => { + entry.rereadTimer = null + if (entry.dropped || this._closing || this._groupServed.get(key) !== entry) return + entry.reread = true + this._cutGroupCore(key, entry) + }, delay) + if (entry.rereadTimer.unref) entry.rereadTimer.unref() + } + + // A writer its diary cut (docs/protocol.md section 10) keeps its blocks + // below the cut and, from the cut on, its group-leave and group-key blocks + // only, which every fold still counts there and readers read past a hole: + // a reader fed by seeders folds as one holding the whole log. The blocks + // past the cut are fetched a chunk at a time, read, and cleared unless + // they are one of those (Hypercore keeps the tree nodes the proofs of the + // kept blocks need). A chunk is listed as read before its blocks go: after + // a crash in between, the next serve reads the range again and clears + // what it still holds beyond the kept blocks. A chunk read from sparse + // peers only (_groupPastRange 'sparse') joins the binding's unsure range, + // read again chunk by chunk once a peer joins and at growing intervals: + // a leave or key block those peers lacked comes from a fuller one, and a + // chunk read whole leaves the range. The quota counts what is kept. One + // pass at a time; growth or a new cut during a pass runs another. + _cutGroupCore (key, entry) { + if (entry.cutting) { + entry.cutAgain = true + if (entry.wake !== null) entry.wake() + return + } + entry.cutting = true + const stale = () => entry.dropped || this._closing || this._groupServed.get(key) !== entry + const run = async () => { + for (;;) { + entry.cutAgain = false + if (stale()) return + const cut = entry.cut + const b = this._groups.get(key) + if (cut === null || !b || !Number.isSafeInteger(b.clearedFrom)) return + const core = entry.core + const below = keptBelow(cut) + const moved = () => stale() || entry.cut !== cut + if (!entry.counted) { + const { keep, clear } = await cutRange(core, b.clearedFrom, b.clearedTo) + if (moved()) continue + for (const [start, end] of clear) await core.clear(start, end) + if (moved()) { + this._rearmGroupDownload(entry) + continue + } + entry.keptPast = new Map(keep) + entry.keptPastBytes = keep.reduce((n, [, bytes]) => n + bytes, 0) + entry.counted = true + } + if (core.byteLength > this.maxLogBytes) { + const to = Math.min(core.contiguousLength, below) + while (entry.belowCounted < to) { + let raw = null + try { + raw = await core.get(entry.belowCounted, { wait: false, raw: true }) + } catch {} + if (moved()) break + entry.belowBytes += raw === null ? 0 : raw.byteLength + entry.belowCounted++ + } + if (moved()) continue + } + if (this._groupKeptOver(entry)) { + // Refused as at admission, or every rescan would fetch it again. + const owner = this._served.get(entry.diary) + if (owner && !owner.dropped) owner.groupRefused.set(key, b.seq) + this._dropGroup(key, 'quota') + return + } + if (!entry.cutConfirmed) return + // A lower cut first reads down from the range already read, so that + // range stays one piece; then up to the signed length; then, when + // asked, the unsure range again. + const end = Math.min(core.length, MAX_LOG_BLOCKS) + const down = b.clearedFrom > below + const unsure = Number.isSafeInteger(b.unsureFrom) + let from + let to + let again = false + if (down) { + from = Math.max(below, b.clearedFrom - CUT_CHUNK) + to = b.clearedFrom + } else if (b.clearedTo < end) { + from = b.clearedTo + to = Math.min(end, from + CUT_CHUNK) + } else if (entry.reread && unsure) { + again = true + from = entry.rereadAt === null ? b.unsureFrom : Math.max(entry.rereadAt, b.unsureFrom) + to = Math.min(b.unsureTo, from + CUT_CHUNK) + } else { + entry.reread = false + return + } + const res = from < to ? await this._groupPastRange(entry, from, to) : 'done' + if (moved()) continue + if (res === null) { + if (entry.cutAgain) continue + return + } + const { keep, clear } = await cutRange(core, from, to) + if (moved()) continue + if (again) { + if (res === 'sparse') entry.rereadLeft = widen(entry.rereadLeft, from, to) + entry.rereadAt = to + if (to >= b.unsureTo) { + const left = entry.rereadLeft + if (left === null) { + delete b.unsureFrom + delete b.unsureTo + } else { + b.unsureFrom = left.from + b.unsureTo = left.to + } + entry.reread = false + entry.rereadAt = null + entry.rereadLeft = null + entry.rereadDelay = Math.min(entry.rereadDelay * 2, REREAD_MAX_MS) + this._armReread(key, entry, entry.rereadDelay) + this._persist().catch(noop) + } + } else { + if (down) b.clearedFrom = from + else b.clearedTo = to + if (res === 'sparse') { + const u = widen(unsure ? { from: b.unsureFrom, to: b.unsureTo } : null, from, to) + b.unsureFrom = u.from + b.unsureTo = u.to + if (entry.rereadAt !== null) entry.rereadLeft = widen(entry.rereadLeft, from, to) + if (!unsure) this._armReread(key, entry, entry.rereadDelay) + } + try { + await this._persist() + } catch (err) { + entry.counted = false + throw err + } + if (moved()) { + entry.counted = false + continue + } + } + for (const [start, end] of clear) await core.clear(start, end) + if (moved()) { + entry.counted = false + this._rearmGroupDownload(entry) + continue + } + for (const [i, bytes] of keep) { + if (entry.keptPast.has(i)) continue + entry.keptPast.set(i, bytes) + entry.keptPastBytes += bytes + } + } + } + run().catch(noop).then(() => { + entry.cutting = false + if (entry.cutAgain && !stale()) this._cutGroupCore(key, entry) + }) + } + // Local re-gate of a served group core, the twin of _gateMedia: block 0 // in the main store must be byte-identical to the quarantined one (else // 'fork') and a header naming the same keys. @@ -4760,7 +5305,7 @@ class Seeder { if (raw === null) return 'header-invalid' if (entry.expected !== null && !b4a.equals(raw, entry.expected)) return 'fork' visibility = this._groupVisibility(entry.group) - return groupVerdict(raw, key, entry.group, entry.diary, visibility) + return groupVerdict(raw, key, entry.group, entry.diary, visibility, entry.cut === 0) } check().then(async (reason) => { if (this.regateHook) await this.regateHook(key, reason) @@ -4770,7 +5315,7 @@ class Seeder { // _recheckGroupBoxes skips a writer not gated yet: check the box // against the visibility known now. const now = this._groupVisibility(entry.group) - if (reason === null && now !== visibility) reason = groupVerdict(raw, key, entry.group, entry.diary, now) + if (reason === null && now !== visibility) reason = groupVerdict(raw, key, entry.group, entry.diary, now, entry.cut === 0) if (reason !== null) return this._dropGroup(key, reason) entry.headerOk = true entry.expected = null @@ -4968,7 +5513,7 @@ class Seeder { raw = await entry.core.get(0, { wait: false, raw: true }) } catch {} if (raw === null) return null - return groupVerdict(raw, key, entry.group, entry.diary, visibility) + return groupVerdict(raw, key, entry.group, entry.diary, visibility, entry.cut === 0) } checks.push(check().then((reason) => { if (reason === null || entry.dropped || this._closing || this._groupServed.get(key) !== entry) return @@ -8894,5 +9439,6 @@ module.exports = { DISCOVERY_VIEW_MAX_KEYS, // For tests. liveGroupJoins, - groupWalkStep + groupWalkStep, + groupWalkCut } diff --git a/packages/core/lib/sparse-held.js b/packages/core/lib/sparse-held.js new file mode 100644 index 00000000..1660b8eb --- /dev/null +++ b/packages/core/lib/sparse-held.js @@ -0,0 +1,194 @@ +'use strict' + +// A seeder keeps a cut writer sparse (#350, docs/protocol.md "Sparse +// copies"): its blocks below the cut, then only its leaves and key blocks. +// A reader holding such a copy is as caught up as its peers let it be once +// every block a connected peer advertises below the signed length is held. +// Hypercore (10 and 11) keeps what a peer advertises per replicator Peer +// (core.peers, as deletion.js advertisedBy reads it): its contiguous range, +// then its bitfield. Past that range the bitfield only arrives as the answer +// to the want a download range sends: one want per bitfield segment, which +// the peer answers with one message from the segment's first block up to +// the last page it has in it. Until that answer is here a block the peer +// holds there reads as not advertised: such a peer has not answered yet, +// and a goal that trusted it would end before the block came. +// The owner of a cut writer reads the same per-peer state the other way: +// a seeder holds its leaves and key blocks past the cut once it advertises +// them or was served them (askHeld, peerHolds). +// Browser-safe: no Node built-ins. + +const { decodeBlock } = require('./records') + +// RemoteBitfield's segment size, the same in Hypercore 10 and 11: a want +// asks for one segment. +const BITFIELD_SEGMENT = 2097152 + +// Whether `peer` has told this core what it holds of [from, end) past its +// contiguous range: it has synced, and either holds nothing past that range +// below end or the answer for each bitfield segment of it arrived. That +// answer always carries the segment's first page; a later page the peer +// holds nothing in is never sent, so its absence once the first page is +// here means nothing held there, not a missing answer. +function peerAnswered (peer, end, from = 0) { + if (peer.remoteSynced !== true) return false + const start = Math.max(from, peer.remoteContiguousLength) + const to = Math.min(end, peer.remoteLength || 0) + if (start >= to) return true + for (let s = Math.floor(start / BITFIELD_SEGMENT); s * BITFIELD_SEGMENT < to; s++) { + if (peer.remoteBitfield.getBitfield(s * BITFIELD_SEGMENT) === null) return false + } + return true +} + +// The first block of [from, end) `peer` advertises, or -1. +function nextAdvertised (peer, from, end) { + if (from < peer.remoteContiguousLength) return from < end ? from : -1 + const bits = peer.remoteBitfield + if (typeof bits.firstSet === 'function') { + const i = bits.firstSet(from) + return i >= 0 && i < end ? i : -1 + } + for (let i = from; i < end; i++) if (bits.get(i) === true) return i + return -1 +} + +// -> true once some connected peer of `core` has synced, every synced peer +// has answered for the blocks of [from, end) (peerAnswered) and every block +// of it one of them advertises is held. A peer that has not synced yet is +// not waited for. Never throws (false on any error). +async function advertisedHeld (core, end, from = 0) { + try { + const peers = [...core.peers].filter((p) => p.remoteSynced === true) + if (peers.length === 0) return false + for (const peer of peers) if (!peerAnswered(peer, end, from)) return false + const start = Math.max(from, core.contiguousLength) + for (const peer of peers) { + let i = nextAdvertised(peer, start, end) + while (i !== -1) { + if (!(await core.has(i))) return false + i = nextAdvertised(peer, i + 1, end) + } + } + return true + } catch { + return false + } +} + +// What `core` holds, as Hypercore 10 and 11 keep it: core.core.bitfield, the +// structure has() itself reads for an open default session. null for any +// other session (a snapshot, a batch), which then asks has() per block. +function heldBits (core) { + try { + if (core.opened !== true) return null + if (core.state && typeof core.state.isDefault === 'function' && !core.state.isDefault()) return null + const bits = core.core && core.core.bitfield + return bits && typeof bits.firstSet === 'function' ? bits : null + } catch { + return null + } +} + +// The first block of [from, end) `core` holds, or -1: a sparse copy past +// its hole is mostly empty, and a scan of has() per block would cost its +// signed length. +async function nextHeld (core, from, end) { + const bits = heldBits(core) + if (bits !== null) { + const i = bits.firstSet(from) + return i >= 0 && i < end ? i : -1 + } + for (let i = from; i < end; i++) if (await core.has(i)) return i + return -1 +} + +// Whether `peer` advertises a block of [from, to) `core` does not hold, one +// this core could still fetch from it. +async function peerOffers (core, peer, from, to) { + for (let i = nextAdvertised(peer, from, to); i !== -1; i = nextAdvertised(peer, i + 1, to)) { + if (!(await core.has(i))) return true + } + return false +} + +// Whether `peer` holds every block in `seqs`: it advertises it, or this +// core served it to that peer (`served`, a Set of indexes, or null). A +// peer never announces a block it gets to one that holds it already, so a +// full copy sees what it served this way only. +function peerHolds (peer, seqs, served = null) { + for (const seq of seqs) { + if (seq < peer.remoteContiguousLength) continue + if (peer.remoteBitfield.get(seq) !== true && (served === null || !served.has(seq))) return false + } + return true +} + +// A want handle that never resolves anything: the want stays for the +// peer's life, so blocks the peer gets there later are announced too. +const STANDING_WANT = { addWant () {}, removeWant () {} } + +// Asks `peer` for its bitfield segments holding `seqs`, once per segment. +// A full copy sends no wants of its own, and a peer tells a core what it +// holds past its contiguous range only in answer to one (Hypercore 11 also +// announces a block it gets only to peers that want its segment, or near +// the tip). Hypercore internals, as a download range calls them: 10's +// Peer#_maybeWant takes a range, 11's a want handle. +function askHeld (peer, seqs) { + if (typeof peer._maybeWant !== 'function') return + for (const seq of seqs) { + if (seq < peer.remoteContiguousLength) continue + try { + if (peer.wants && typeof peer.wants.add === 'function') peer._maybeWant(seq, STANDING_WANT) + else peer._maybeWant(seq, 1) + } catch {} + } +} + +// The blocks of [from, to) of a full local copy that a seeder keeps past a +// cut: its group-leave and group-key blocks (seeder.js keptPastCut). +async function keptPastSeqs (core, from, to) { + const out = [] + for (let seq = from; seq < to; seq++) { + let block = null + try { + block = decodeBlock(await core.get(seq, { wait: false, raw: true })) + } catch {} + if (block !== null && (block.type === 'group-leave' || block.type === 'group-key')) out.push(seq) + } + return out +} + +// The runs of blocks of [from, to) `core` holds, as [start, end) pairs, or +// null past `max` runs. Without the bitfield (heldBits null) it asks has() +// per block, and only over at most `max` blocks (null past that). +async function heldRuns (core, from, to, max) { + const runs = [] + const bits = heldBits(core) + if (bits === null) { + if (to - from > max) return null + let start = -1 + for (let i = from; i < to; i++) { + if (await core.has(i)) { + if (start === -1) start = i + } else if (start !== -1) { + runs.push([start, i]) + start = -1 + } + } + if (start !== -1) runs.push([start, to]) + return runs.length > max ? null : runs + } + let i = from + while (i < to) { + const start = bits.firstSet(i) + if (start === -1 || start >= to) break + if (runs.length === max) return null + let end = bits.firstUnset(start) + if (end === -1 || end > to) end = to + runs.push([start, end]) + i = end + } + return runs +} + +module.exports = { advertisedHeld, peerAnswered, peerOffers, nextHeld, heldRuns, peerHolds, askHeld, keptPastSeqs, BITFIELD_SEGMENT } diff --git a/packages/core/lib/store.js b/packages/core/lib/store.js index 855fbeaf..143ebea0 100644 --- a/packages/core/lib/store.js +++ b/packages/core/lib/store.js @@ -9,6 +9,7 @@ const sodium = require('sodium-universal') const Corestore = require('corestore') const { validateRecord, decodeBlock, MAX_BLOCK_BYTES, MAX_LOG_BLOCKS, MAX_LOG_BYTES } = require('./records') const { presentState } = require('./deletion') +const { nextHeld } = require('./sparse-held') // Private directory: created 0o700 and tightened to 0o700 when it exists. // Follows symlinks: the data dir itself may be a symlink the user chose. @@ -520,6 +521,30 @@ async function readBlocks (core) { return out } +// A group writer's blocks as readBlocks, then past the first hole only the +// leaves and key blocks held: a seeder clears the rest of a cut writer +// (#350), and the fold reads past a hole only what a cut keeps. +async function readHeldBlocks (core) { + assertOpen(core) + const limit = Math.min(core.length, MAX_LOG_BLOCKS) + const out = [] + let hole = false + for (let i = 0; i < limit; i++) { + const block = (await core.has(i)) ? await core.get(i, { wait: false, raw: true }) : null + if (block === null) { + hole = true + const next = await nextHeld(core, i + 1, limit) + if (next === -1) break + i = next - 1 + continue + } + const record = decodeBlock(block) + if (hole && (record === null || (record.type !== 'group-leave' && record.type !== 'group-key'))) continue + out.push({ seq: i, record }) + } + return out +} + function assertOpen (core) { if (core.closing) { const err = new Error('SESSION_CLOSED: cannot read from a closed session') @@ -935,6 +960,7 @@ module.exports = { readAll, readRaw, readBlocks, + readHeldBlocks, appendRecord, appendRecords, readConfig, diff --git a/packages/core/lib/sync.js b/packages/core/lib/sync.js index 2301df44..6a70886b 100644 --- a/packages/core/lib/sync.js +++ b/packages/core/lib/sync.js @@ -73,7 +73,8 @@ const { pageLifecycle, originOf, verifyClose, verifyMoved } = require('./page-li const { certForKey, seederCertForKey } = require('./cert') const { createBlocklistState, verifySnapshot, compareSnapshots } = require('./blocklist') const { attachBlocklist } = require('./blocklist-net') -const { validateGroupHeader, validateWriterHeader, buildCommunity, groupJoinHistory, groupCutHistory, groupConfirmHistory, MAX_GROUP_WRITERS, MAX_GROUPS_PER_DIARY, MAX_GROUP_WRITERS_PER_DIARY, MAX_FROZEN_WRITERS_PER_DIARY, MAX_WRITER_RELEASES_PER_DIARY, MAX_GROUPS_PER_COMMUNITY } = require('./group') +const { validateGroupHeader, validateWriterHeader, buildCommunity, groupJoinHistory, groupCutHistory, groupConfirmHistory, diaryWriterCut, MAX_GROUP_WRITERS, MAX_GROUPS_PER_DIARY, MAX_GROUP_WRITERS_PER_DIARY, MAX_FROZEN_WRITERS_PER_DIARY, MAX_WRITER_RELEASES_PER_DIARY, MAX_GROUPS_PER_COMMUNITY } = require('./group') +const { advertisedHeld, keptPastSeqs, askHeld, peerHolds } = require('./sparse-held') const { attachGroups, GROUPS_LIMIT_MAX, GROUPS_REQUEST_TIMEOUT_MS, MESSAGE_BURST: GROUPS_BURST, MESSAGE_RATE: GROUPS_RATE } = require('./groups-net') const { presentState, @@ -138,6 +139,12 @@ const MAX_WRITER_PAGES = Math.ceil(MAX_GROUP_WRITERS / GROUPS_LIMIT_MAX) // the socket dropped for 'message-rate' (that would cut diary replication). const GROUP_REQUEST_BURST = Math.floor(GROUPS_BURST / 8) const GROUP_REQUEST_RATE = GROUPS_RATE / 2 +// A foreign writer held past its prefix is checked again this often while +// its goal waits on what its peers advertise (sparse-held.js). +const SPARSE_CHECK_MS = 100 +// An own cut writer whose leaves or key blocks a seeder does not advertise +// yet is checked again this often (its answer to the want raises no event). +const KEPT_CHECK_MS = 250 // Page items whose diary or join seq a gate disproved, per seeder, before // this run stops paging that seeder. const MAX_BAD_HINTS = 32 @@ -2164,7 +2171,15 @@ async function runSync ({ continue } seen.add(g.group) - ownWriters.set(g.writer, { group: g.group, core }) + // uploaded: replicator Peer -> the block indexes served to it. + const own = { group: g.group, core, cut: null, kept: null, keptReading: false, keptTimer: null, uploaded: new WeakMap() } + core.on('upload', (index, byteLength, from) => { + if (!from) return + let set = own.uploaded.get(from) + if (set === undefined) own.uploaded.set(from, (set = new Set())) + set.add(index) + }) + ownWriters.set(g.writer, own) groupList.push({ group: g.group, writer: g.writer, nonce: g.nonce }) groupSet.add(g.group) if (g.kind === 'community' || (g.writer === g.group && await ownRootKind(core) === 'community')) ownCommunities.add(g.group) @@ -2185,6 +2200,17 @@ async function runSync ({ } catch {} } } + // A writer the own diary cut is pushed whole up to the cut, and past it + // only its leaves and key blocks, all a seeder keeps there + // (docs/protocol.md "Sparse copies"). + if (ownWriters.size > 0) { + try { + const entries = await readAll(diary) + for (const [w, own] of ownWriters) if (w !== own.group) own.cut = diaryWriterCut(entries, own.group, w) + } catch { + log('aviso: falha ao ler os cortes do diário') + } + } // A joined group's root stored by an earlier run names its community // before any seeder lists it again. for (const g of groupList) { @@ -2647,7 +2673,15 @@ async function runSync ({ if (it && it.length > pushed) pushed = it.length } const own = ownWriters.get(g.writer) - if (own.core.length > pushed) unmet.push(g.writer + ':' + own.core.length + '/' + pushed) + // A writer its diary cut is kept to the cut, block 0 always: past it + // a seeder holds only its leaves and key blocks, never the whole + // length, and those must be there too. + const goal = own.cut === null ? own.core.length : Math.min(own.core.length, Math.max(own.cut, 1)) + if (goal > pushed) unmet.push(g.writer + ':' + goal + '/' + pushed) + else if (goal < own.core.length) { + const missing = ownKeptMissing(own, seeders, g, goal) + if (missing !== -1) unmet.push(g.writer + ':corte ' + missing) + } } for (const [key, advertised] of groupTargets) { if (ownWriters.has(key)) continue @@ -2657,9 +2691,12 @@ async function runSync ({ const active = e && e.state === 'active' const ours = active ? e.core.contiguousLength : 0 const want = active ? Math.max(advertised, e.core.length) : advertised + // A copy past the advertised prefix with a hole (a writer a seeder keeps + // sparse) is met once its peers' advertised blocks are all held. + const short = !active || (ours < want && (ours < advertised || !sparseMet(e))) // Until the main store's block 0 is compared with the gated one, ending // the run would skip that comparison and the growth it reports. - if (!active || ours < want || e.expected !== null) unmet.push(key + ':' + ours + '/' + want) + if (short || e.expected !== null) unmet.push(key + ':' + ours + '/' + want) if (active) { const d = groupDiaries.get(e.diary) if (d && !d.failed && (d.present < d.want || d.expected !== null)) unmet.push(e.diary + ':' + d.present + '/' + d.want) @@ -2668,6 +2705,80 @@ async function runSync ({ return unmet } + // The own leaves and key blocks of cut writer `own` past its cut, read + // once per length (null while that read runs). + function ownKept (own) { + const length = Math.min(own.core.length, maxLogBlocks) + const k = own.kept + if (k !== null && k.from === own.cut && k.to === length) return k.seqs + if (own.keptReading || ended) return null + own.keptReading = true + const from = own.cut + const start = k !== null && k.from === from && k.to < length ? k.to : from + const before = start === from ? [] : k.seqs + track(keptPastSeqs(own.core, start, length), (seqs) => { + own.keptReading = false + own.kept = { from, to: length, seqs: before.concat(seqs) } + evaluate() + }, () => { + own.keptReading = false + }) + return null + } + + // The first own leave or key block of cut writer `own` that no verified + // seeder listing it at `goal` holds (-1 when one holds them all): past + // the cut a seeder keeps those blocks only, so the writer is pushed once + // one holds them. Each such seeder is asked for its bitfield there (a + // full core sends no wants of its own) and checked again soon. + function ownKeptMissing (own, seeders, g, goal) { + const seqs = ownKept(own) + if (seqs === null) return own.cut + if (seqs.length === 0) return -1 + let missing = seqs[0] + for (const p of seeders) { + const pass = p.groupPasses.get(g.group) + const it = pass && pass.items.get(g.writer) + if (!it || it.length < goal) continue + const peer = own.core.peers.find((x) => x.stream === p.socket) + if (!peer) continue + const served = own.uploaded.get(peer) || null + if (peerHolds(peer, seqs, served)) return -1 + askHeld(peer, seqs) + missing = seqs.find((seq) => !peerHolds(peer, [seq], served)) + } + if (own.keptTimer === null && !ended) { + own.keptTimer = later(() => { + own.keptTimer = null + evaluate() + }, KEPT_CHECK_MS) + } + return missing + } + + // Whether foreign writer e, held up to a hole past what its seeders list, + // holds every block a connected peer advertises below its signed length, + // from the last check at its present length and prefix (sparse-held.js). + // A missing or stale check runs again shortly; evaluate() follows it. + function sparseMet (e) { + const s = e.sparse + if (s && s.length === e.core.length && s.contiguous === e.core.contiguousLength && s.done) return true + if (e.sparseTimer || ended) return false + e.sparseTimer = later(() => { + e.sparseTimer = null + const core = e.core + if (ended || core === null || e.state !== 'active') return + const length = core.length + const contiguous = core.contiguousLength + track(advertisedHeld(core, Math.min(length, maxLogBlocks)), (done) => { + if (e.core !== core) return + e.sparse = { length, contiguous, done } + evaluate() + }, noop) + }, SPARSE_CHECK_MS) + return false + } + function groupSockets (peer) { const out = [] if (!peer.socket.destroyed) out.push(peer.socket) @@ -3056,6 +3167,8 @@ async function runSync ({ diaryRaw: null, // D's block 0, judged again when an issuer is revoked wasStored: false, purging: null, + sparse: null, // { length, contiguous, done }: the last advertisedHeld check + sparseTimer: null, accepted: false // gated as a writer of this group (now or in an earlier run) } groupCores.set(item.key, e) diff --git a/packages/core/test/group-member-6.test.js b/packages/core/test/group-member-6.test.js index 5a03fa7c..20a9ec5a 100644 --- a/packages/core/test/group-member-6.test.js +++ b/packages/core/test/group-member-6.test.js @@ -9,6 +9,7 @@ const { readBoundRoots } = require('../lib/sync') const store = require('../lib/store') const member = require('../lib/member') const { hex, person, exchange, pair, writerEntries, offlineAppend, rejects, FILL, createGroup, joinGroup, THUMBS, HEART } = require('./helpers/group-member') +const { groupCutRecord } = require('../lib/records') test('loading a group records a root its creator bound, and not one it never bound', async (t) => { const { b, group } = await pair(t) @@ -217,3 +218,44 @@ test('group.read leaves out the fold\'s forum keys: no topic, poll voters or thr const hidden = await member.groupRead(a, { group }) t.alike(keys(hidden), [['Sim!', false, false, false], ['', false, false, false]], 'a hidden root shows no poll') }) + +// A seeder keeps a cut writer sparse (#350): below the cut, then only its +// leaves and key blocks. A node holding such a copy folds it as the whole one. +test('a cut writer held sparse folds like the whole copy (#350)', async (t) => { + const { a, b, group } = await pair(t) + await member.groupPost(b, { group, text: 'antes' }) + const own = await writerEntries(b, group) + const cut = own.length + await offlineAppend(b, group, groupLib.groupPost({ text: 'depois' })) + await offlineAppend(b, group, groupLib.groupPost({ text: 'depois 2' })) + const leave = await offlineAppend(b, group, groupLib.groupLeave({ seen: null, cuts: [{ writer: own.key, seq: cut + 2 }] })) + const s = await store.openStore(b.dataDir) + try { + await store.appendRecord(await store.openDiary(s), groupCutRecord({ group, writer: own.key, cut })) + } finally { + await s.close() + } + await exchange(b, a) + const view = async () => { + const info = await member.groupInfo(a, { group }) + const page = await member.groupRead(a, { group }) + const members = await member.groupMembers(a, { group }) + return JSON.stringify({ info, members, posts: page.items.map((p) => [p.writer, p.seq, p.text]) }) + } + const whole = await view() + t.ok(JSON.parse(whole).posts.some((p) => p[2] === 'antes'), 'control: the post below the cut counts') + t.absent(JSON.parse(whole).posts.some((p) => p[2] === 'depois'), 'control: the posts past the cut do not') + t.absent(JSON.parse(whole).members.members.some((m) => m.member === b.key), 'control: the leave past the cut counts') + const sa = await store.openStore(a.dataDir) + try { + const core = sa.get({ key: Buffer.from(own.key, 'hex'), writable: false }) + await core.ready() + await core.clear(cut, leave.seq) + t.is(core.contiguousLength, cut, 'a sparse copy: a hole at the cut') + t.ok(await core.has(leave.seq), 'the leave past it is held') + await core.close() + } finally { + await sa.close() + } + t.is(await view(), whole, 'the sparse copy folds like the whole one') +}) diff --git a/packages/core/test/group-private.test.js b/packages/core/test/group-private.test.js index 124e5f04..4535a115 100644 --- a/packages/core/test/group-private.test.js +++ b/packages/core/test/group-private.test.js @@ -570,6 +570,59 @@ test('a key block citing a cut community writer stays readable (#350)', (t) => { t.is(g.rekeyPending, true) }) +test('a key block citing a cut CW held sparse is judged by its signed length (#350)', (t) => { + const w = pworld({ cadmins: [B] }) + w.add(B, wB) + w.comm.push(cwKey(B), communityBan({ member: K(0xabc), on: true, cuts: [], at: AT(5) })) + const ban = w.push(G, groupBan({ member: Y, on: true, cuts: [{ writer: wY, seq: 1 }], at: AT(6) })) + const E2 = w.rotate(wB, [G, wA, wB, wX], { basis: [ban], auth: w.cauth(B), at: 7 }) + sealedPost(w, wX, E2, 8) + w.comm.mark(B, cwKey(B), 1) + const whole = w.input(w.reader(wX)) + const cwB = whole.community.writers.find((l) => l.key === cwKey(B)) + t.is(cwB.entries.length, 2, 'the auth cites the whole CW, past its cut') + const trimmed = (signed) => ({ ...whole, community: { ...whole.community, writers: whole.community.writers.map((l) => l === cwB ? { ...l, entries: l.entries.slice(0, 1), signed } : l) } }) + t.alike(buildGroup(trimmed(undefined)).posts, [], 'control: held alone, the trimmed CW is too short for the auth') + const g = buildGroup(trimmed(2)) + t.alike(g.posts.map((p) => p.text), ['p8'], 'the signed length keeps the history readable') + t.is(g.epoch, null, 'and still sends nothing') + t.is(normalize(g), normalize(buildGroup(whole)), 'the sparse CW folds like the whole one') +}) + +test('a cut writer held sparse folds like the whole copy; a copy with a hole folds as its prefix (#350)', (t) => { + const w = pworld() + const E2 = w.rotate(wA, [G, wA, wX, wY], { at: 6 }) + sealedPost(w, wA, E2, 7) + const cut = w.len(wA) + sealedPost(w, wA, E2, 8) + const E3 = w.rotate(wA, [G, wA, wX, wY], { at: 9 }) + sealedPost(w, wA, E3, 10) + w.push(wA, groupLeave({ cuts: [{ writer: wA, seq: w.len(wA) }], at: AT(11) })) + sealedPost(w, wX, E3, 12) + sealedPost(w, wX, E3, 13) + const signed = w.len(wA) + const marks = new Map([[A, [{ writer: wA, cut, seq: 2 }]]]) + const base = w.input(w.reader(wX), { marks }) + const kept = (e) => e.seq < cut || e.record.type === 'group-leave' || e.record.type === 'group-key' + const withCopy = (key, entries, extra = {}) => ({ ...base, writers: base.writers.map((l) => l.key === key ? { ...l, entries, ...extra } : l) }) + const wAentries = base.writers.find((l) => l.key === wA).entries + const wXentries = base.writers.find((l) => l.key === wX).entries + const whole = buildGroup(base) + t.alike(whole.posts.map((p) => p.text), ['p7', 'p12', 'p13'], 'control: the whole copy drops the posts past the cut') + t.is(whole.members.get(A), 'left', 'control: the leave past the cut counts') + const sparse = wAentries.filter(kept) + t.alike(sparse.map((e) => e.seq), [0, 1, 2, 4, 6], 'the sparse copy: below the cut, the key block and the leave past it') + t.is(normalize(buildGroup(withCopy(wA, sparse, { signed }))), normalize(whole), 'the sparse copy folds like the whole one') + t.is(normalize(buildGroup({ ...base, writers: [...base.writers, { key: wA, diary: A, entries: sparse, signed }] })), normalize(whole), 'and so does a sparse copy next to the whole one') + // A copy that stops below the cut counts only up to its hole, even with + // the leave and key block held past it. + const short = wAentries.filter((e) => e.seq < 2 || e.seq === 4 || e.seq === 6) + t.is(normalize(buildGroup(withCopy(wA, short, { signed }))), normalize(buildGroup(withCopy(wA, wAentries.slice(0, 2)))), 'a hole below the cut: the prefix') + // An uncut writer never reads past a hole. + const holed = wXentries.filter((e) => e.seq !== 1) + t.is(normalize(buildGroup(withCopy(wX, holed))), normalize(buildGroup(withCopy(wX, wXentries.slice(0, 1)))), 'an uncut writer: the prefix') +}) + test('a diary cut below its own leave keeps the diary left and the removal (#350)', (t) => { const unleave = (cut) => { const w = pworld() diff --git a/packages/core/test/helpers/seeder-groups.js b/packages/core/test/helpers/seeder-groups.js index 0fe2005f..26b50131 100644 --- a/packages/core/test/helpers/seeder-groups.js +++ b/packages/core/test/helpers/seeder-groups.js @@ -127,7 +127,7 @@ async function member (t, bootstrap, assoc, fill, { certOf = (key) => validCert( }) await joinAndFlush(swarm, [topicFor(assoc.hex)]) const seederPeer = () => until(() => peers.find((p) => p.remoteRole === 'seeder'), { name: 'hello do semeador' }) - return { store, diary, key, cores, swarm, messages, seederPeer, certOf } + return { store, diary, key, cores, swarm, messages, peers, seederPeer, certOf } } async function admit (m, assoc) { diff --git a/packages/core/test/seeder-delete.test.js b/packages/core/test/seeder-delete.test.js index cce2dc28..6d093511 100644 --- a/packages/core/test/seeder-delete.test.js +++ b/packages/core/test/seeder-delete.test.js @@ -1414,6 +1414,7 @@ function groupFixture (held, blocks = GROUP_BLOCKS) { groupWalks: new Map(), groupEvents: new Map(), groupLive: new Map(), + groupRefused: new Map(), recent: [], core: { fork: 0, @@ -1480,6 +1481,20 @@ test('a deleted group-cut that arrives after the scan steps the slot walk at its t.alike(entry.groupLive.get(CUT_WRITERS[8]), { group: GROUP_KEY, seq: 10, left: false }, 'the ninth takes the slot it freed') }) +test('a deleted cut that arrives after the scan tries again the join it lowers the cut of (#350)', async (t) => { + const held = new Set(CUT_BLOCKS.keys()) + const { seeder, entry } = groupFixture(held, CUT_BLOCKS) + for (const seq of [1, 2, 3, 4, 5, 6, 7, 8, 10]) await seeder._pageArrived(GROUP_DIARY, entry, seq) + // Refused at their joins (a boxless writer of a private group, a core + // past its quota): only a lower cut on the writer lifts that. + entry.groupRefused.set(CUT_WRITERS[0], 1) + entry.groupRefused.set(CUT_WRITERS[1], 2) + + await seeder._pageArrived(GROUP_DIARY, entry, 9) + t.absent(entry.groupRefused.has(CUT_WRITERS[0]), 'the writer the late cut released is tried again') + t.is(entry.groupRefused.get(CUT_WRITERS[1]), 2, 'a writer it does not cut stays refused') +}) + // A member joined a writer (seq 1), left (seq 2, the join off) and deleted // the join (seq 3). The scan passed the join as a hole: when it arrives the // walk counts the writer, which readers then check by that on join. diff --git a/packages/core/test/seeder-groups-cuts.test.js b/packages/core/test/seeder-groups-cuts.test.js index 83e3b949..7ea1fbe8 100644 --- a/packages/core/test/seeder-groups-cuts.test.js +++ b/packages/core/test/seeder-groups-cuts.test.js @@ -1,19 +1,23 @@ 'use strict' // The writers a diary cut (#350, docs/protocol.md section 10) at a seeder: -// the slot walk picks the writers that count (liveGroupJoins), and a frozen -// or released writer is held whole, as a reader folds from a contiguous -// copy (a reduced copy comes with the client nodes' sparse read). +// the slot walk picks the writers that count (liveGroupJoins), a frozen +// writer is kept below its cut plus its leaves and key blocks, a released +// one as a stub (block 0, leaves, key blocks), the quota counts what is +// kept, and a restart recomputes both from admitted.json and the scan. const fs = require('node:fs') const path = require('node:path') const test = require('brittle') const b4a = require('b4a') -const { testnet, until } = require('./helpers/net') -const { Seeder, liveGroupJoins, groupWalkStep } = require('../lib/seeder') +const { testnet, until, localSwarmOptions } = require('./helpers/net') +const { createSwarm, attachAdmission } = require('../lib/net') +const { openStore } = require('../lib/store') +const { Seeder, liveGroupJoins, groupWalkStep, groupWalkCut } = require('../lib/seeder') const { groupJoinRecord, groupCutRecord, + deleteRecord, MAX_GROUP_WRITERS_PER_DIARY, MAX_FROZEN_WRITERS_PER_DIARY, MAX_WRITER_RELEASES_PER_DIARY, @@ -34,7 +38,8 @@ const { rejected, downloaded, readGroups, - scansDone + scansDone, + gate } = require('./helpers/seeder-groups') const G = 'aa'.repeat(32) @@ -80,6 +85,9 @@ test('liveGroupJoins: a cut frees a slot for a 9th writer, never past 8 live, 8 t.ok(released.has(key(8)), 'so does a release') t.alike(released.get(key(0)), { group: G, seq: 1, left: true }, 'and the released writer stays, left') t.is(diary([...eight, cut(key(0), 0)]).walks.get(G).state.get(key(0)), 'released') + t.is(groupWalkCut(diary([...eight, cut(key(0), 0)]).walks, G, key(0)), 0, 'kept at cut 0') + t.is(groupWalkCut(diary([...eight, cut(key(1), 2)]).walks, G, key(1)), 2, 'a frozen writer at its cut') + t.is(groupWalkCut(diary(eight).walks, G, key(0)), null, 'a live writer has none') // Eight frozen, then eight live: the 17th never counts. const full = [] @@ -134,14 +142,16 @@ test('a stored binding the scan no longer lists gets its on join too, never an o const H = 'bb'.repeat(32) const W = key(0) const { decls, walks } = diary([join(W), join(W, false), join(W, true, H)]) - const entry = { gated: true, dropped: false, groupDecls: decls, groupWalks: walks, groupBound: new Map(), groupRefused: new Map(), groupLive: liveGroupJoins(decls, walks) } + // A diary the scan read to its length (its certificate and three records), + // as _groupCutScan reads it on every reconcile. + const entry = { gated: true, dropped: false, scanned: 4, core: { length: 4 }, groupDecls: decls, groupWalks: walks, groupBound: new Map(), groupRefused: new Map(), groupLive: liveGroupJoins(decls, walks) } t.is(entry.groupLive.get(W).group, H, 'the scan lists W in H only') const rows = [] const seeder = Object.create(Seeder.prototype) Object.assign(seeder, { _closing: false, _groups: new Map([[W, { diary: D, group: G, seq: 2 }]]), - _groupServed: new Map([[W, {}]]), + _groupServed: new Map([[W, { dropped: false, cut: null, cutConfirmed: false }]]), _groupIndex: { addWriter: (group, k, d, seq) => rows.push({ group, key: k, diary: d, seq }), setLeft: (k, left) => rows.push({ key: k, left }) @@ -273,7 +283,7 @@ async function blocksHeld (core, from, to) { return out } -test('a cut writer is held whole: its blocks past the cut stay, and a fresh reader reads its leave in order (#350)', async (t) => { +test('a cut after the download clears from the cut, keeps only leaves and key blocks past it, and serves what it keeps with proofs (#350)', async (t) => { t.timeout(150000) const { bootstrap } = await testnet(t) const assoc = makeAssociation() @@ -285,32 +295,43 @@ test('a cut writer is held whole: its blocks past the cut stay, and a fresh read await admit(B, assoc) await admitted(events, W) await downloaded(seeder, W, 6) + t.absent('cut' in readGroups(dataDir)[W], 'no cut yet') - // B cuts W at 3, then keeps writing to it: a post, a leave, another post. await B.diary.append(groupCutRecord({ group: R, writer: W, cut: 3 })) - await settledFor(seeder, B) + const entry = seeder._groupServed.get(W) + await until(async () => entry.cut === 3 && !entry.cutting && seeder._groups.get(W).clearedTo === 6 && (await blocksHeld(entry.core, 0, 6)).length === 3, { name: 'W cortado em 3' }) + t.alike(await blocksHeld(entry.core, 0, 6), [0, 1, 2], 'only the blocks below the cut are stored') + t.is(entry.core.contiguousLength, 3) + t.alike(readGroups(dataDir)[W], { diary: B.key, group: R, seq: 1, cut: 3, clearedFrom: 3, clearedTo: 6 }, 'the cut and the range read are persisted with the binding') + t.ok(seeder._groupServed.has(W), 'the writer stays served') + + // B keeps writing past its cut: a post goes, a leave and a key block stay + // (the fold counts them past a cut), and so does the next post's gap. await B.cores.w.append(b4a.from(JSON.stringify(groupPost({ text: 'depois do corte' })))) await B.cores.w.append(b4a.from(JSON.stringify(groupLeave({ cuts: [{ writer: W, seq: 7 }] })))) + await B.cores.w.append(b4a.from(JSON.stringify({ type: 'group-key' }))) await B.cores.w.append(b4a.from(JSON.stringify(groupPost({ text: 'mais um' })))) - await downloaded(seeder, W, 9) - const entry = seeder._groupServed.get(W) - t.alike(await blocksHeld(entry.core, 0, 9), [0, 1, 2, 3, 4, 5, 6, 7, 8], 'every block stays, past the cut too') - t.absent('cut' in readGroups(dataDir)[W], 'no cut is kept with the binding') + await until(async () => entry.core.length === 10 && !entry.cutting && seeder._groups.get(W).clearedTo === 10, { name: 'W lido depois do corte' }) + t.alike(await blocksHeld(entry.core, 0, 10), [0, 1, 2, 7, 8], 'below the cut, then the leave and the key block only') + t.is(entry.keptPast.size, 2, 'the quota counts the two blocks kept past the cut') - // B goes; a reader that dials the seeder only reads W from block 0 up, - // as the nodes do (store.js readBlocks stops at the first hole). + // B goes; a reader that dials the seeder only verifies the kept blocks + // against the signed length (Hypercore checks each proof). await B.swarm.destroy() const r = await reader(t, bootstrap, publicKey) const copy = r.store.get({ key: b4a.from(W, 'hex') }) await copy.ready() - await until(() => copy.length === 9, { name: 'comprimento assinado no leitor', timeoutMs: 30000 }) - const types = [] - for (let i = 0; i < 9; i++) types.push(JSON.parse(b4a.toString(await copy.get(i, { timeout: 15000 }))).type) - t.is(types[7], 'group-leave', 'the leave past the cut is reached by a contiguous read') + await until(() => copy.length === 10, { name: 'comprimento assinado no leitor', timeoutMs: 30000 }) + for (let i = 0; i < 3; i++) t.ok(await copy.get(i, { timeout: 15000 }), 'block ' + i + ' verified from the seeder') + const leave = JSON.parse(b4a.toString(await copy.get(7, { timeout: 15000 }))) + t.is(leave.type, 'group-leave', 'the leave past the cut is served, verified') + t.is(JSON.parse(b4a.toString(await copy.get(8, { timeout: 15000 }))).type, 'group-key', 'and the key block') + await t.exception(copy.get(3, { timeout: 3000 }), 'a post past the cut is not served') + await t.exception(copy.get(9, { timeout: 3000 }), 'nor one after the leave') t.is(events.filter((e) => e.event === 'fatal').length, 0) }) -test('a 9th writer after a freeze is admitted, a counted writer whose join is off is admitted as left, the 10th never (#350)', async (t) => { +test('a 9th writer after a freeze is admitted at its cut, a counted writer whose join is off is admitted as left, the 10th never (#350)', async (t) => { t.timeout(240000) const { bootstrap } = await testnet(t) const assoc = makeAssociation() @@ -340,9 +361,12 @@ test('a 9th writer after a freeze is admitted, a counted writer whose join is of await settledFor(seeder, B) t.absent(events.some((e) => e.key === W[9]), 'the 10th is past the cap and never tried') t.absent(W[9] in readGroups(dataDir)) - await downloaded(seeder, W[0], 2) - t.pass('the frozen writer is held whole') + t.is(readGroups(dataDir)[W[0]].cut, 1, 'the frozen writer is listed at its cut') + await until(() => seeder._groups.get(W[0]).clearedTo === 2 && !seeder._groupServed.get(W[0]).cutting, { name: 'W0 lido depois do corte' }) + t.is(seeder._groupServed.get(W[0]).core.contiguousLength, 1, 'W0 below its cut') + t.is(await seeder._groupServed.get(W[0]).core.has(1), false, 'and holds nothing past it') await downloaded(seeder, W[8], 2) + t.absent('cut' in readGroups(dataDir)[W[8]], 'the 9th has no cut') const rows = new Map(seeder._groupIndex.page(R).items.filter((i) => i.diary === B.key).map((i) => [i.key, i])) t.is(rows.size, 9, 'nine writers of B listed') @@ -391,7 +415,7 @@ test('a writer that left before its admission is listed by its on join, and a bi t.is(first.events.filter((e) => e.event === 'fatal').length + second.events.filter((e) => e.event === 'fatal').length, 0) }) -test('a writer its diary released (cut 0) stays held whole, and a fresh reader reads its leave in order (#350)', async (t) => { +test('a writer its diary released (cut 0) is kept as a stub that serves its leave to a fresh reader (#350)', async (t) => { t.timeout(150000) const { bootstrap } = await testnet(t) const assoc = makeAssociation() @@ -408,28 +432,28 @@ test('a writer its diary released (cut 0) stays held whole, and a fresh reader r await downloaded(seeder, W, 8) await B.diary.append(groupCutRecord({ group: R, writer: W, cut: 0 })) - await settledFor(seeder, B) const entry = seeder._groupServed.get(W) - t.ok(entry, 'still served') - t.alike(await blocksHeld(entry.core, 0, 8), [0, 1, 2, 3, 4, 5, 6, 7], 'every block stays') - t.absent('cut' in readGroups(dataDir)[W], 'no cut is kept with the binding') - t.ok(seeder._groupIndex.writer(W), 'it stays in the index') + await until(async () => entry.cut === 0 && !entry.cutting && seeder._groups.get(W).clearedTo === 8 && (await blocksHeld(entry.core, 0, 8)).length === 2, { name: 'W reduzido a um toco' }) + t.alike(await blocksHeld(entry.core, 0, 8), [0, 6], 'block 0 and the leave only') + t.is(readGroups(dataDir)[W].cut, 0, 'the stub is persisted at cut 0') + t.ok(seeder._groupIndex.writer(W), 'and stays in the index') t.absent(events.some((e) => e.event === 'group-rejected' && e.key === W), 'never purged') + await settledFor(seeder, B) t.ok(seeder._groupServed.has(R), 'the root stays') - // B goes; a reader that dials the seeder only reads W from block 0 up. + // B goes; a reader that dials the seeder only gets the leave, verified. await B.swarm.destroy() const r = await reader(t, bootstrap, publicKey) const copy = r.store.get({ key: b4a.from(W, 'hex') }) await copy.ready() await until(() => copy.length === 8, { name: 'comprimento assinado no leitor', timeoutMs: 30000 }) - const types = [] - for (let i = 0; i < 8; i++) types.push(JSON.parse(b4a.toString(await copy.get(i, { timeout: 15000 }))).type) - t.is(types[6], 'group-leave', 'the leave of a released writer is reached by a contiguous read') + t.ok(await copy.get(0, { timeout: 15000 }), 'block 0 verified from the seeder') + t.is(JSON.parse(b4a.toString(await copy.get(6, { timeout: 15000 }))).type, 'group-leave', 'the leave of a released writer is served, verified') + await t.exception(copy.get(1, { timeout: 3000 }), 'a post is not served') t.is(events.filter((e) => e.event === 'fatal').length, 0) }) -test('a restart keeps a cut writer and a released one whole (#350)', async (t) => { +test('a restart restores a writer at its persisted cut and the scan lowers it or makes it a stub (#350)', async (t) => { t.timeout(240000) const { bootstrap } = await testnet(t) const assoc = makeAssociation() @@ -446,8 +470,10 @@ test('a restart keeps a cut writer and a released one whole (#350)', async (t) = await admitted(first.events, W2) await settledFor(first.seeder, B) // Held before the restart, or the restore drops them as headerless. - await downloaded(first.seeder, W, 6) + await until(() => first.seeder._groups.get(W).clearedTo === 6 && !first.seeder._groupServed.get(W).cutting, { name: 'W lido depois do corte' }) + t.alike(await blocksHeld(first.seeder._groupServed.get(W).core, 0, 6), [0, 1, 2, 3]) await downloaded(first.seeder, W2, 2) + t.is(readGroups(dataDir)[W].cut, 4) await first.seeder.close() // While the seeder is down B lowers W's cut and releases W2. @@ -455,41 +481,337 @@ test('a restart keeps a cut writer and a released one whole (#350)', async (t) = await B.diary.append(groupCutRecord({ group: R, writer: W2, cut: 0 })) const second = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex }) - t.ok(second.seeder._groupServed.get(W), 'W served at ready') - await settledFor(second.seeder, B) - t.alike(await blocksHeld(second.seeder._groupServed.get(W).core, 0, 6), [0, 1, 2, 3, 4, 5], 'W whole') - t.alike(await blocksHeld(second.seeder._groupServed.get(W2).core, 0, 2), [0, 1], 'W2 released, whole') + const entry = second.seeder._groupServed.get(W) + t.ok(entry, 'W served at ready') + t.ok(entry.cut === 4 || entry.cut === 2, 'from its persisted cut') + await until(async () => entry.cut === 2 && !entry.cutting && (await blocksHeld(entry.core, 0, 6)).length === 2, { name: 'W cortado em 2' }) + t.alike(await blocksHeld(entry.core, 0, 6), [0, 1]) + t.is(readGroups(dataDir)[W].cut, 2, 'the lower cut is persisted') + t.is(readGroups(dataDir)[W].clearedFrom, 2, 'and the range read reaches down to it') + const stub = second.seeder._groupServed.get(W2) + await until(async () => stub.cut === 0 && !stub.cutting && (await blocksHeld(stub.core, 0, 2)).length === 1, { name: 'W2 reduzido a um toco' }) + t.alike(await blocksHeld(stub.core, 0, 2), [0], 'W2 released: its block 0 stays') + await until(() => readGroups(dataDir)[W2].cut === 0, { name: 'toco persistido' }) t.absent(second.events.some((e) => e.event === 'group-rejected' && (e.key === W || e.key === W2)), 'never purged') t.is(second.events.filter((e) => e.event === 'fatal').length, 0) }) -test('a cut writer is held to its whole log\'s quota: refused past it at admission, dropped once it grows past it (#350)', async (t) => { +test('a cut its diary deleted still applies at the seeder, after a restart too (#350)', async (t) => { + t.timeout(240000) + const { bootstrap } = await testnet(t) + const assoc = makeAssociation() + const { dataDir } = prepareSeederDir(t, assoc) + const first = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex }) + + // B's diary: certificate, the join of W, then the cut at 3 (block 2). + const { A, B, R, W } = await cutWorld(t, bootstrap, assoc, (R, W) => [groupCutRecord({ group: R, writer: W, cut: 3 })]) + await admit(A, assoc) + await admit(B, assoc) + await admitted(first.events, W) + await until(async () => first.seeder._groups.get(W).clearedTo === 6 && !first.seeder._groupServed.get(W).cutting, { name: 'W lido depois do corte' }) + t.alike(await blocksHeld(first.seeder._groupServed.get(W).core, 0, 6), [0, 1, 2], 'cleared past its cut') + + // B deletes the cut record: a group-cut ignores deletes (docs/protocol.md + // "Deletion"), so the seeder keeps both the block and the cut. + await B.diary.append(deleteRecord({ seq: 2 })) + await settledFor(first.seeder, B) + t.ok(await first.seeder._served.get(B.key).core.has(2), 'the deleted cut stays at the seeder') + t.is(first.seeder._groups.get(W).cut, 3, 'and so does the cut') + await first.seeder.close() + + const second = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex }) + const entry = second.seeder._groupServed.get(W) + t.ok(entry, 'W served at ready') + await settledFor(second.seeder, B) + t.is(entry.cut, 3, 'the scan reads the deleted cut again') + t.alike(readGroups(dataDir)[W], { diary: B.key, group: R, seq: 1, cut: 3, clearedFrom: 3, clearedTo: 6 }, 'the binding keeps it') + t.alike(await blocksHeld(entry.core, 0, 6), [0, 1, 2], 'and nothing it cleared is fetched again') + t.is(first.events.filter((e) => e.event === 'fatal').length + second.events.filter((e) => e.event === 'fatal').length, 0) +}) + +test('a persisted cut the diary does not give is dropped once the scan caught up, and what clearing removed is fetched again (#350)', async (t) => { + t.timeout(240000) + const { bootstrap } = await testnet(t) + const assoc = makeAssociation() + const { dataDir } = prepareSeederDir(t, assoc) + const first = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex }) + + // B's diary: certificate and the join of W, no cut. + const { A, B, R, W } = await cutWorld(t, bootstrap, assoc) + await admit(A, assoc) + await admit(B, assoc) + await admitted(first.events, W) + await downloaded(first.seeder, W, 6) + await first.seeder.close() + + // A cut stored with W's binding and cleared past it, as admitted.json + // keeps one whose record the scan no longer gives (a store restored from + // an older copy, say): a deleted cut still counts since #717, so a + // delete cannot make one go. + const file = path.join(dataDir, 'admitted.json') + const data = JSON.parse(fs.readFileSync(file, 'utf8')) + Object.assign(data.groups[W], { cut: 3, clearedFrom: 3, clearedTo: 6 }) + fs.writeFileSync(file, JSON.stringify(data)) + const store = await openStore(dataDir) + const core = store.get({ key: b4a.from(W, 'hex') }) + await core.ready() + await core.clear(3, 6) + t.alike(await blocksHeld(core, 0, 6), [0, 1, 2], 'cleared past the stored cut') + await core.close() + await store.close() + + const second = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex }) + const entry = second.seeder._groupServed.get(W) + t.ok(entry, 'W served at ready') + t.is(entry.cut, 3, 'from its persisted cut') + await until(async () => entry.cut === null && (await blocksHeld(entry.core, 0, 6)).length === 6, { name: 'W buscado de novo' }) + t.alike(await blocksHeld(entry.core, 0, 6), [0, 1, 2, 3, 4, 5], 'every block fetched again') + await until(() => !('cut' in readGroups(dataDir)[W]), { name: 'sem corte persistido' }) + t.alike(readGroups(dataDir)[W], { diary: B.key, group: R, seq: 1 }, 'the binding has no cut nor range') + t.absent(second.events.some((e) => e.event === 'group-rejected' && e.key === W), 'never purged') + t.is(second.events.filter((e) => e.event === 'fatal').length, 0) +}) + +test('a cut writer counts what it keeps toward the log quota: posts past the cut are cleared, not counted (#350)', async (t) => { t.timeout(150000) const { bootstrap } = await testnet(t) const assoc = makeAssociation() const { dataDir } = prepareSeederDir(t, assoc) const { seeder, events } = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex, maxLogBytes: 20000 }) - // A raw block near the block limit is enough to cross the byte quota. + // Blocks past a cut are only read at the seeder, never validated: a raw + // block near the block limit is enough to cross the byte quota. const big = () => b4a.from(JSON.stringify({ type: 'group-post', text: 'x'.repeat(12000) })) - // W is cut at 2 and already holds 24 KB past its cut when B is admitted. + const bigKey = () => b4a.from(JSON.stringify({ type: 'group-key', pad: 'x'.repeat(12000) })) + + // W is cut at 2 and already holds 24 KB of posts past its cut when B is + // admitted: they are cleared, so W is served. const { A, B, R, W } = await cutWorld(t, bootstrap, assoc, (R, W) => [groupCutRecord({ group: R, writer: W, cut: 2 })]) await B.cores.w.append(big()) await B.cores.w.append(big()) await admit(A, assoc) await admit(B, assoc) - t.is((await rejected(events, W, 'quota')).reason, 'quota', 'refused at admission: the bytes past the cut count') - t.absent(seeder._groupServed.has(W)) - - // W2, cut at 2 while small, is served; it then grows past the quota. + await admitted(events, W) + const entry = seeder._groupServed.get(W) + await until(async () => !entry.cutting && seeder._groups.get(W).clearedTo === 8 && (await blocksHeld(entry.core, 0, 8)).length === 2, { name: 'W lido depois do corte' }) + t.alike(await blocksHeld(entry.core, 0, 8), [0, 1], 'the posts past the cut are cleared') + t.absent(events.some((e) => e.event === 'group-rejected' && e.key === W), 'and not counted') + + // Two key blocks past the cut stay, and they are past the quota. + await B.cores.w.append(bigKey()) + await B.cores.w.append(bigKey()) + t.is((await rejected(events, W, 'quota')).reason, 'quota', 'dropped once what it keeps passes the quota') + await until(() => !seeder._groupServed.has(W), { name: 'W fora' }) + + // W2, cut at 4 while small, is served; it then grows past the quota + // below its cut. const W2 = await addWriter(B, R, 'w2') await B.diary.append(groupJoinRecord({ group: R, writer: W2, on: true })) - await B.diary.append(groupCutRecord({ group: R, writer: W2, cut: 2 })) + await B.diary.append(groupCutRecord({ group: R, writer: W2, cut: 4 })) await admitted(events, W2) await downloaded(seeder, W2, 2) - t.ok(seeder._groupServed.has(W2), 'served while within the quota') + await until(() => seeder._groupServed.get(W2).cut === 4, { name: 'W2 em 4' }) await B.cores.w2.append(big()) await B.cores.w2.append(big()) - t.is((await rejected(events, W2, 'quota')).reason, 'quota', 'dropped once its log passes the quota') + t.is((await rejected(events, W2, 'quota')).reason, 'quota', 'the bytes below the cut count') await until(() => !seeder._groupServed.has(W2), { name: 'W2 fora' }) + await settledFor(seeder, B) + t.is(events.filter((e) => e.event === 'group-admitted' && e.key === W).length, 1, 'a writer dropped for what it keeps is not fetched again') t.is(events.filter((e) => e.event === 'fatal').length, 0) }) + +test('a lower cut seen while a writer core is still opening is the cut it is served at (#350)', async (t) => { + t.timeout(150000) + const { bootstrap } = await testnet(t) + const assoc = makeAssociation() + const { dataDir } = prepareSeederDir(t, assoc) + const hold = gate() + const { seeder, events } = await startSeeder(t, bootstrap, dataDir, { + associationKey: assoc.hex, + admitHook: (key, step) => step === 'open' ? hold.hook(key) : undefined + }) + t.teardown(() => hold.open()) + + const { A, B, R, W } = await cutWorld(t, bootstrap, assoc, (R, W) => [groupCutRecord({ group: R, writer: W, cut: 4 })]) + hold.key = W + await admit(A, assoc) + await admit(B, assoc) + await until(() => hold.entered, { name: 'W abrindo' }) + t.is(seeder._groups.get(W).cut, 4, 'admitted at its cut') + await B.diary.append(groupCutRecord({ group: R, writer: W, cut: 2 })) + await until(() => seeder._groups.get(W).cut === 2, { name: 'corte menor na ligação', timeoutMs: 30000 }) + t.absent(seeder._groupServed.has(W), 'while the core is still opening') + await until(() => readGroups(dataDir)[W].cut === 2, { name: 'corte menor persistido' }) + hold.open() + await admitted(events, W) + const entry = seeder._groupServed.get(W) + t.is(entry.cut, 2, 'served at the lower cut') + await until(async () => !entry.cutting && seeder._groups.get(W).clearedTo === 6 && (await blocksHeld(entry.core, 0, 6)).length === 2, { name: 'W abaixo de 2' }) + t.alike(await blocksHeld(entry.core, 0, 6), [0, 1]) + t.is(events.filter((e) => e.event === 'fatal').length, 0) +}) + +test('a boxless writer of a private group its diary released is kept as a stub; before the release it is refused (#350)', async (t) => { + t.timeout(180000) + const { bootstrap } = await testnet(t) + const assoc = makeAssociation() + const { dataDir, publicKey } = prepareSeederDir(t, assoc) + const first = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex }) + + let P = null + const A = await member(t, bootstrap, assoc, async ({ key, make }) => { + P = await make('root', groupHeader({ diary: key, visibility: 'private', join: 'approval', name: null, box: 'e'.repeat(64) })) + return [groupJoinRecord({ group: P, writer: P, on: true })] + }) + // N: a writer of B in the private group without a box (one its diary + // replaced), header and a post. + let N = null + const B = await member(t, bootstrap, assoc, async ({ key, make }) => { + N = await make('n', writerHeader({ group: P, diary: key }), [groupPost({ text: 'oi' })]) + return [groupJoinRecord({ group: P, writer: N, on: true })] + }) + await admit(A, assoc) + await admitted(first.events, P) + await until(() => first.seeder._groupVisibility(P) === 'private', { name: 'raiz privada conhecida' }) + await admit(B, assoc) + await rejected(first.events, N, 'header-invalid') + t.absent(first.seeder._groupServed.has(N), 'refused while counted') + + await B.diary.append(groupCutRecord({ group: P, writer: N, cut: 0 })) + await admitted(first.events, N) + const entry = first.seeder._groupServed.get(N) + await until(async () => !entry.cutting && first.seeder._groups.get(N).clearedTo === 2 && (await blocksHeld(entry.core, 0, 2)).length === 1, { name: 'N reduzido a um toco' }) + t.alike(await blocksHeld(entry.core, 0, 2), [0], 'released: its block 0 is kept') + t.is(readGroups(dataDir)[N].cut, 0) + + // A reader fetches the stub's header from the seeder alone. + await B.swarm.destroy() + const r = await reader(t, bootstrap, publicKey) + const copy = r.store.get({ key: b4a.from(N, 'hex') }) + await copy.ready() + const header = JSON.parse(b4a.toString(await copy.get(0, { timeout: 20000 }))) + t.is(header.type, 'group-writer') + t.is(header.box, null, 'the boxless header readers reject') + await first.seeder.close() + + // A restart keeps the stub: its header is judged as a released writer's. + const second = await startSeeder(t, bootstrap, dataDir, { associationKey: assoc.hex }) + t.ok(second.seeder._groupServed.get(N), 'N served at ready') + t.absent(second.events.some((e) => e.event === 'group-rejected' && e.key === N), 'not purged') + t.is(first.events.filter((e) => e.event === 'fatal').length + second.events.filter((e) => e.event === 'fatal').length, 0) +}) + +test('a seeder fed only by a seeder that keeps a writer sparse reads past cut + 256, then reads it again once a fuller peer joins (#350)', async (t) => { + t.timeout(240000) + const { bootstrap } = await testnet(t) + const assoc = makeAssociation() + const first = prepareSeederDir(t, assoc) + const one = await startSeeder(t, bootstrap, first.dataDir, { associationKey: assoc.hex }) + + // W: header, five posts, 300 more, a leave and one more post; cut at 3, + // so the leave sits past the first chunk read after the cut. + const { A, B, R, W } = await cutWorld(t, bootstrap, assoc) + const posts = [] + for (let i = 0; i < 300; i++) posts.push(b4a.from(JSON.stringify(groupPost({ text: 'mais ' + i })))) + await B.cores.w.append(posts) + const leaveAt = B.cores.w.length + await B.cores.w.append(b4a.from(JSON.stringify(groupLeave({ cuts: [{ writer: W, seq: leaveAt }] })))) + await B.cores.w.append(b4a.from(JSON.stringify(groupPost({ text: 'depois da saída' })))) + const length = B.cores.w.length + t.ok(leaveAt >= 3 + 256, 'the leave is past the first chunk') + await B.diary.append(groupCutRecord({ group: R, writer: W, cut: 3 })) + await admit(A, assoc) + await admit(B, assoc) + await admitted(one.events, W) + const e1 = one.seeder._groupServed.get(W) + await until(async () => e1.cut === 3 && !e1.cutting && one.seeder._groups.get(W).clearedTo === length && (await blocksHeld(e1.core, 0, length)).length === 4, { name: 'W esparso no primeiro semeador' }) + t.alike(await blocksHeld(e1.core, 0, length), [0, 1, 2, leaveAt], 'the first seeder keeps W below its cut and its leave') + t.absent('unsureFrom' in readGroups(first.dataDir)[W], 'read whole from its owner') + + // B goes: a second seeder reads W from the first only, which holds + // nothing of the first chunk past the cut. A carries B's diary to it (the + // gate fetches a diary from its announcer), never W. + const carried = A.store.get({ key: B.diary.key }) + await carried.ready() + await carried.download({ start: 0, end: B.diary.length }).done() + await B.swarm.destroy() + const second = prepareSeederDir(t, assoc) + const two = await startSeeder(t, bootstrap, second.dataDir, { associationKey: assoc.hex }) + const twoKey = hex(two.seeder.swarm.keyPair.publicKey) + const peer = await until(() => A.peers.find((p) => p.remoteRole === 'seeder' && p.peerKey === twoKey), { name: 'segundo semeador' }) + peer.send({ type: 'announce', key: A.key, cert: A.certOf(A.key) }) + peer.send({ type: 'announce', key: B.key, cert: B.certOf(B.key) }) + await admitted(two.events, W) + const e2 = two.seeder._groupServed.get(W) + await until(async () => !e2.cutting && two.seeder._groups.get(W).clearedTo === length && await e2.core.has(leaveAt), { name: 'W lido do primeiro semeador', timeoutMs: 60000 }) + t.alike(await blocksHeld(e2.core, 0, length), [0, 1, 2, leaveAt], 'the second seeder keeps the leave too') + t.is(e2.keptPast.size, 1, 'and counts it toward the quota') + const listed = readGroups(second.dataDir)[W] + t.alike([listed.unsureFrom, listed.unsureTo], [3, length], 'the range read from sparse peers only is listed as such') + + // A peer that joins with nothing there the second seeder lacks (A opens + // W, empty) reads nothing again and leaves the backoff. The backoff is + // pushed out first, so that only a peer could start a pass. + const far = 5 * 60 * 1000 + e2.rereadDelay = far + two.seeder._armReread(W, e2, far) + await until(() => !e2.cutting && !e2.reread, { name: 'nenhuma releitura em curso' }) + let passes = 0 + const pastRange = two.seeder._groupPastRange + two.seeder._groupPastRange = function (...args) { + passes++ + return pastRange.apply(this, args) + } + let joined = false + e2.core.once('peer-add', () => { joined = true }) + const empty = A.store.get({ key: b4a.from(W, 'hex') }) + await empty.ready() + t.teardown(() => empty.close(), { force: true }) + await until(() => joined, { name: 'A ligado a W' }) + await new Promise((resolve) => setTimeout(resolve, 3000)) + t.is(passes, 0, 'a peer offering nothing there starts no pass') + t.is(e2.rereadDelay, far, 'and leaves the backoff') + delete two.seeder._groupPastRange + + // B comes back with W whole, which it advertises: the range is read again + // at once, each chunk whole, and what is not kept is cleared again. + const back = createSwarm(localSwarmOptions(bootstrap)) + t.teardown(() => back.destroy(), { force: true }) + back.on('connection', (socket) => { + socket.on('error', () => {}) + B.store.replicate(socket) + attachAdmission(socket, { role: 'member', have: [] }) + }) + back.joinPeer(b4a.from(twoKey, 'hex')) + await until(() => !('unsureFrom' in two.seeder._groups.get(W)) && !e2.cutting, { name: 'W relido inteiro', timeoutMs: 60000 }) + t.alike(await blocksHeld(e2.core, 0, length), [0, 1, 2, leaveAt], 'what it read again past the cut is cleared again') + t.absent('unsureFrom' in readGroups(second.dataDir)[W], 'and the range is listed as read whole') + t.is(one.events.filter((e) => e.event === 'fatal').length + two.events.filter((e) => e.event === 'fatal').length, 0) +}) + +// A drop or a close destroys the range of a chunk past the cut, whose done() +// then never settles, and wakes only a wait already armed: one landing while +// the loop checks what its peers advertise must still end it. +test('a drop during the check of a chunk past the cut ends its wait (#350)', async (t) => { + const listeners = new Set() + const entry = { past: null, wake: null, dropped: false } + const peer = { remoteSynced: true, remoteContiguousLength: 300, remoteLength: 300, remoteBitfield: { getBitfield: () => null, get: () => false } } + const core = { + peers: [peer], + contiguousLength: 3, + download: () => ({ done: () => new Promise(() => {}), destroy () {} }), + // The drop lands while advertisedHeld reads what is held, and the close + // takes the peers with it. + async has () { + entry.dropped = true + core.peers = [] + return false + }, + on (name, fn) { listeners.add(fn) }, + off (name, fn) { listeners.delete(fn) } + } + entry.core = core + const read = Seeder.prototype._groupPastRange.call({ _closing: false }, entry, 3, 259) + const ended = await Promise.race([read.then((value) => ({ value })), new Promise((resolve) => setTimeout(() => resolve(null), 2000))]) + t.alike(ended, { value: null }, 'the wait ends with nothing read') + t.is(listeners.size, 0, 'and listens on the core no more') +}) diff --git a/packages/core/test/sparse-held.test.js b/packages/core/test/sparse-held.test.js new file mode 100644 index 00000000..666e5156 --- /dev/null +++ b/packages/core/test/sparse-held.test.js @@ -0,0 +1,119 @@ +'use strict' + +// Sparse copies against bare Hypercores (#350, lib/sparse-held.js): a peer +// that holds a few blocks of a long log answers for all of it, and a held +// block past a hole is found without asking has() for every block. + +const test = require('brittle') +const b4a = require('b4a') +const Hypercore = require('hypercore') +const path = require('node:path') +const { tmpDir } = require('./helpers/tmp') +const { advertisedHeld, nextHeld, peerHolds, peerOffers } = require('../lib/sparse-held') +const { readHeldBlocks } = require('../lib/store') + +function link (a, b) { + const left = a.replicate(true) + const right = b.replicate(false) + left.pipe(right).pipe(left) + return () => { + left.destroy() + right.destroy() + } +} + +async function open (t, dir, name, key) { + const core = new Hypercore(path.join(dir, name), key) + await core.ready() + t.teardown(() => core.close(), { force: true }) + return core +} + +// W: a log of `length` blocks, `leaveAt` a group-leave, the rest posts. +// S holds [0, 10) and the leave, fetched from W, which then goes. +async function sparseSource (t, length, leaveAt) { + const dir = tmpDir(t) + const w = await open(t, dir, 'w') + const blocks = [] + for (let i = 0; i < length; i++) { + blocks.push(b4a.from(JSON.stringify(i === leaveAt ? { type: 'group-leave' } : { type: 'group-post', text: 'p' + i }))) + } + for (let i = 0; i < length; i += 5000) await w.append(blocks.slice(i, i + 5000)) + const s = await open(t, dir, 's', w.key) + const unlink = link(w, s) + await s.update({ wait: true }) + await s.download({ start: 0, end: 10 }).done() + await s.download({ start: leaveAt, end: leaveAt + 1 }).done() + unlink() + return { dir, key: w.key, s } +} + +test('a peer holding a prefix and one block of a long log answers for all of it (#350)', async (t) => { + t.timeout(60000) + // The leave sits in the first bitfield page; the signed length reaches a + // second page the peer holds nothing in, so it never sends that page. + const { dir, key, s } = await sparseSource(t, 40000, 500) + const r = await open(t, dir, 'r', key) + link(s, r) + await r.update({ wait: true }) + r.download({ start: 0, end: r.length }) + const deadline = Date.now() + 8000 + let held = false + while (!held && Date.now() < deadline) { + held = await advertisedHeld(r, r.length) + if (!held) await new Promise((resolve) => setTimeout(resolve, 100)) + } + t.ok(held, 'every advertised block is held well before the deadline') + t.is(r.contiguousLength, 10) + t.ok(await r.has(500), 'the leave came') + t.absent(await r.has(10), 'nothing the peer does not hold') +}) + +test('a held block past a hole is found without asking has() for every block (#350)', async (t) => { + t.timeout(60000) + const { s } = await sparseSource(t, 40000, 35000) + const real = s.has.bind(s) + let calls = 0 + s.has = (...args) => { + calls++ + return real(...args) + } + t.is(await nextHeld(s, 10, s.length), 35000, 'the next held block') + t.is(await nextHeld(s, 35001, s.length), -1, 'none after it') + calls = 0 + const entries = await readHeldBlocks(s) + t.alike(entries.map((e) => e.seq), [...Array(10).keys(), 35000], 'the prefix and the leave past the hole') + t.ok(calls < 100, 'has() asked ' + calls + ' times, not once per block') +}) + +// The owner of a cut writer counts a block past the cut as at the seeder +// once the seeder's peer advertises it or this core served it to that peer: +// a peer never announces a block it gets to one that holds it already. +test('a peer holds a block it advertises or was served (#350)', (t) => { + const peer = { remoteContiguousLength: 3, remoteBitfield: { get: (i) => i === 5 } } + t.ok(peerHolds(peer, [0, 2, 5]), 'its contiguous range and a block it advertises') + t.absent(peerHolds(peer, [5, 7]), 'not a block it neither advertises nor was served') + t.absent(peerHolds(peer, [5, 7], new Set()), 'nor with nothing served') + t.ok(peerHolds(peer, [5, 7], new Set([7])), 'a block served to it counts') + t.absent(peerHolds(peer, [5, 7, 9], new Set([7])), 'only that one') +}) + +// A seeder reads its unsure range again for a peer that joins only when it +// advertises a block there the seeder lacks. +test('a peer offers a block it advertises and the core lacks (#350)', async (t) => { + const held = new Set([4, 6]) + const core = { has: async (i) => held.has(i) } + const firstOf = (set, from) => { + const next = [...set].filter((i) => i >= from).sort((a, b) => a - b) + return next.length > 0 ? next[0] : -1 + } + const peer = (contiguous, set) => ({ + remoteContiguousLength: contiguous, + remoteBitfield: { get: (i) => set.has(i), firstSet: (from) => firstOf(set, from) } + }) + t.absent(await peerOffers(core, peer(0, new Set()), 3, 10), 'nothing advertised') + t.absent(await peerOffers(core, peer(0, new Set([4, 6])), 3, 10), 'only what the core holds') + t.ok(await peerOffers(core, peer(0, new Set([4, 8])), 3, 10), 'a block the core lacks') + t.absent(await peerOffers(core, peer(0, new Set([4, 12])), 3, 10), 'not past the range') + t.ok(await peerOffers(core, peer(10, new Set()), 3, 10), 'its contiguous range over a block the core lacks') +}) diff --git a/packages/core/test/sync-groups-sparse.test.js b/packages/core/test/sync-groups-sparse.test.js new file mode 100644 index 00000000..d3b5060a --- /dev/null +++ b/packages/core/test/sync-groups-sparse.test.js @@ -0,0 +1,191 @@ +'use strict' + +// A seeder keeps a writer its diary cut sparse (#350, docs/protocol.md +// section 10): below the cut, then only its leaves and key blocks, and a +// released writer as a stub. A fresh member fed by that seeder alone folds +// the group as one holding the whole copies. +const test = require('brittle') +const b4a = require('b4a') +const { testnet } = require('./helpers/net') +const { generateAssociationKeyPair } = require('../lib/cert') +const { groupCutRecord } = require('../lib/records') +const { groupPost, groupLeave } = require('../lib/group') +const store = require('../lib/store') +const member = require('../lib/member') +const { person, exchange, startSeeder, syncOnce, ownWriter, runUntil } = require('./helpers/sync-groups') +const { offlineAppend, writerEntries } = require('./helpers/group-member') + +async function blocksHeld (core, from, to) { + const out = [] + for (let i = from; i < to; i++) if (await core.has(i)) out.push(i) + return out +} + +// The blocks of each writer `ctx` holds, by key. +async function heldCopies (ctx, keys) { + const s = await store.openStore(ctx.dataDir) + try { + const out = {} + for (const key of keys) { + const core = s.get({ key: b4a.from(key, 'hex'), writable: false }) + await core.ready() + out[key] = { length: core.length, contiguous: core.contiguousLength, held: await blocksHeld(core, 0, core.length) } + await core.close() + } + return out + } finally { + await s.close() + } +} + +async function view (ctx, community, groups) { + const out = { + community: { + info: await member.communityInfo(ctx, { community }), + members: await member.communityMembers(ctx, { community }), + groups: await member.communityGroups(ctx, { community }) + } + } + for (const group of groups) { + const info = await member.groupInfo(ctx, { group }) + const members = await member.groupMembers(ctx, { group }) + const page = await member.groupRead(ctx, { group }) + out[group] = { info, members, posts: page.items.map((p) => [p.writer, p.seq, p.text]) } + } + return out +} + +test('a fresh member fed only by a seeder folds a sparse frozen writer and a released stub like the whole copies (#350)', async (t) => { + t.timeout(400000) + const { bootstrap } = await testnet(t) + const kp = generateAssociationKeyPair() + const seeder = await startSeeder(t, bootstrap, kp) + const a = await person(t, kp, 'Ana') + const b = await person(t, kp, 'Bia') + const r = await person(t, kp, 'Rui') + const { community } = await member.communityCreate(a, { name: 'Bairro', join: 'open' }) + const { group: g1 } = await member.groupCreate(a, { name: 'Horta', join: 'open', community }) + const { group: g2 } = await member.groupCreate(a, { name: 'Feira', join: 'open', community }) + await member.groupPost(a, { group: g1, text: 'primeiro' }) + await member.groupPost(a, { group: g2, text: 'bem-vindos' }) + // B and R learn the roots out of band; everything else travels by sync. + await exchange(a, b) + await exchange(a, r) + for (const m of [b, r]) { + await member.communityJoin(m, { community }) + await member.groupJoin(m, { group: g1 }) + await member.groupJoin(m, { group: g2 }) + } + await member.groupPost(b, { group: g1, text: 'antes' }) + await member.groupPost(b, { group: g2, text: 'oi' }) + + // W1, B's writer in G1, is frozen at its length: two posts and a leave + // past the cut. W2, its writer in G2, is released: a post and a leave. + const w1 = ownWriter(b, g1) + const w2 = ownWriter(b, g2) + const cut1 = (await writerEntries(b, g1)).length + await offlineAppend(b, g1, groupPost({ text: 'depois' })) + await offlineAppend(b, g1, groupPost({ text: 'depois 2' })) + const leave1 = await offlineAppend(b, g1, groupLeave({ seen: null, cuts: [{ writer: w1, seq: cut1 + 2 }] })) + const len2 = (await writerEntries(b, g2)).length + await offlineAppend(b, g2, groupPost({ text: 'tchau' })) + const leave2 = await offlineAppend(b, g2, groupLeave({ seen: null, cuts: [{ writer: w2, seq: len2 + 1 }] })) + const s = await store.openStore(b.dataDir) + try { + const diary = await store.openDiary(s) + await store.appendRecord(diary, groupCutRecord({ group: g1, writer: w1, cut: cut1 })) + await store.appendRecord(diary, groupCutRecord({ group: g2, writer: w2, cut: 0 })) + } finally { + await s.close() + } + + const ra = await syncOnce(bootstrap, a) + t.ok(ra.result.ok, 'A pushes its diary and roots: ' + ra.result.errors.join('; ')) + // B's run pushes its diary and writers; it stops once the seeder holds + // both writers sparse. + const sparse = async () => { + const e1 = seeder._groupServed.get(w1) + const e2 = seeder._groupServed.get(w2) + const b1 = seeder._groups.get(w1) + const b2 = seeder._groups.get(w2) + if (!e1 || !e2 || !b1 || !b2 || e1.cutting || e2.cutting) return false + if (b1.cut !== cut1 || b1.clearedTo !== leave1.seq + 1 || b2.cut !== 0 || b2.clearedTo !== leave2.seq + 1) return false + const h1 = await blocksHeld(e1.core, 0, leave1.seq + 1) + const h2 = await blocksHeld(e2.core, 0, leave2.seq + 1) + return h1.length === cut1 + 1 && h2.length === 2 + } + await runUntil(b, bootstrap, sparse) + t.alike(await blocksHeld(seeder._groupServed.get(w1).core, 0, leave1.seq + 1), [...Array(cut1).keys(), leave1.seq], 'the seeder keeps W1 below its cut and its leave') + t.alike(await blocksHeld(seeder._groupServed.get(w2).core, 0, leave2.seq + 1), [0, leave2.seq], 'and W2 as a stub') + // The seeder holds each cut writer up to its cut and its leave: the + // owner's --once counts both pushed. + const rb = await syncOnce(bootstrap, b, { timeoutMs: 30000 }) + t.ok(rb.result.ok, 'B\'s run ends ok before its deadline: ' + rb.result.errors.join('; ')) + + // R, fresh, syncs with the seeder alone (B is offline); its --once ends + // once it holds every block the seeder advertises. + const rr = await syncOnce(bootstrap, r, { timeoutMs: 30000 }) + t.ok(rr.result.ok, 'R\'s run ends ok before its deadline: ' + rr.result.errors.join('; ')) + const copies = await heldCopies(r, [w1, w2]) + t.alike(copies[w1], { length: leave1.seq + 1, contiguous: cut1, held: [...Array(cut1).keys(), leave1.seq] }, 'R holds W1 sparse') + t.alike(copies[w2], { length: leave2.seq + 1, contiguous: 1, held: [0, leave2.seq] }, 'and W2 as a stub') + const fromSeeder = await view(r, community, [g1, g2]) + + // The whole copies, straight from B. + await exchange(b, r) + const whole = await heldCopies(r, [w1, w2]) + t.is(whole[w1].contiguous, leave1.seq + 1, 'control: R now holds W1 whole') + t.is(whole[w2].contiguous, leave2.seq + 1, 'and W2') + const full = await view(r, community, [g1, g2]) + const texts = (g) => full[g].posts.map((p) => p[2]) + t.ok(texts(g1).includes('antes'), 'control: the post below the cut counts') + t.absent(texts(g1).includes('depois'), 'control: a post past the cut does not') + t.absent(texts(g2).includes('oi'), 'control: a released writer counts no post') + t.absent(full[g1].members.members.some((m) => m.member === b.key), 'control: the leave past the cut counts') + t.alike(fromSeeder, full, 'the sparse copies fold like the whole ones') +}) + +test('the owner of a cut writer counts it pushed only once the seeder holds its leave past the cut (#350)', async (t) => { + t.timeout(300000) + const { bootstrap } = await testnet(t) + const kp = generateAssociationKeyPair() + const seeder = await startSeeder(t, bootstrap, kp) + const a = await person(t, kp, 'Ana') + const b = await person(t, kp, 'Bia') + const { community } = await member.communityCreate(a, { name: 'Bairro', join: 'open' }) + const { group } = await member.groupCreate(a, { name: 'Horta', join: 'open', community }) + await exchange(a, b) + await member.communityJoin(b, { community }) + await member.groupJoin(b, { group }) + await member.groupPost(b, { group, text: 'antes' }) + const w = ownWriter(b, group) + const cut = (await writerEntries(b, group)).length + await offlineAppend(b, group, groupPost({ text: 'depois' })) + const leave = await offlineAppend(b, group, groupLeave({ seen: null, cuts: [{ writer: w, seq: cut + 1 }] })) + const s = await store.openStore(b.dataDir) + try { + await store.appendRecord(await store.openDiary(s), groupCutRecord({ group, writer: w, cut })) + } finally { + await s.close() + } + t.ok((await syncOnce(bootstrap, a)).result.ok, 'A pushes its diary and roots') + + // The seeder holds W below the cut but has not read past it yet. + const read = seeder._groupPastRange + let resume = null + const stalled = new Promise((resolve) => { resume = resolve }) + seeder._groupPastRange = async function (...args) { + await stalled + return read.apply(this, args) + } + const first = await syncOnce(bootstrap, b, { timeoutMs: 15000 }) + const errors = first.result.errors.join('; ') + t.absent(first.result.ok, 'B\'s run does not count W pushed at its cut alone') + t.ok(errors.includes(w + ':corte ' + leave.seq), 'it waits for the leave: ' + errors) + t.absent(await seeder._groupServed.get(w).core.has(leave.seq), 'control: the seeder lacks the leave') + + resume() + const second = await syncOnce(bootstrap, b, { timeoutMs: 30000 }) + t.ok(second.result.ok, 'once the seeder reads past the cut the run ends ok: ' + second.result.errors.join('; ')) + t.ok(await seeder._groupServed.get(w).core.has(leave.seq), 'with the leave at the seeder') +}) diff --git a/packages/web-member/lib/member-ops.js b/packages/web-member/lib/member-ops.js index 12fc2edb..93478077 100644 --- a/packages/web-member/lib/member-ops.js +++ b/packages/web-member/lib/member-ops.js @@ -105,6 +105,7 @@ const { engagementState, isLiked, isReposted } = require('@varanda/core/lib/enga const { verifySnapshot, blockedSet, revokedIssuerSet } = require('@varanda/core/lib/blocklist') const { blockedKeys } = require('@varanda/core/lib/member-block') const { UsageError } = require('@varanda/core/lib/errors') +const { heldRuns } = require('@varanda/core/lib/sparse-held') const { copyLinkCard } = require('@varanda/core/lib/link-card') const { sealOffer, commitment } = require('@varanda/core/lib/testimonials') const grp = require('@varanda/core/lib/group') @@ -2695,6 +2696,16 @@ async function groupCoreState (core) { return { entries: (await store.readBlocks(core)).slice(0, length), length, signed, fork } } +// A writer as groupCoreState, and past its first hole the leaves and key +// blocks held below the signed length reported (store.readHeldBlocks): a +// seeder keeps a cut writer sparse (#350). +async function writerCoreState (core) { + const length = core.contiguousLength + const signed = core.length + const fork = core.fork + return { entries: (await store.readHeldBlocks(core)).filter((e) => e.seq < signed), length, signed, fork } +} + // A stored group core's entries (store.readBlocks: no diary deletion fold), // read-only -> { entries, length, fork } or null. async function readGroupCore (ctx, key, rec = null) { @@ -2711,8 +2722,8 @@ async function readGroupCore (ctx, key, rec = null) { const MEMO_MAX = 64 const MEMO_TTL_MS = 30000 -// A replica with more missing blocks than this above its contiguous length -// is not signed: its stamp would cost as much as the read. +// A replica holding more runs of blocks than this past its contiguous +// length is not signed: its stamp would cost as much as the read. const MEMO_MAX_HOLES = 4096 // Tests only: VARANDA_MEMO_CHECK=1 turns the check mode on for every ctx // and gives a ctx without a memo one per storage. @@ -2741,18 +2752,22 @@ function memoChecks (ctx) { return ctx.memoCheck === true || MEMO_ENV_CHECK } +// A writer is read past its first hole too (groupCoreState), so a group +// core's stamp is a replica's. function groupStamp (core) { - return core.contiguousLength + ',' + core.length + ',' + core.fork + return replicaStamp(core) } // readVisible also shows the stored blocks past a hole, so a replica's -// stamp names each of them (null: too many to sign). +// stamp names the runs of them (null: too many to sign). A long stub or a +// writer cut early holds a few runs far past its prefix. async function replicaStamp (core) { const contiguous = core.contiguousLength const length = core.length + const runs = await heldRuns(core, contiguous, length, MEMO_MAX_HOLES) + if (runs === null) return null let out = contiguous + ',' + length + ',' + core.fork - if (length - contiguous > MEMO_MAX_HOLES) return null - for (let i = contiguous; i < length; i++) if (await core.has(i)) out += '.' + i + for (const [start, end] of runs) out += '.' + start + '-' + end return out } @@ -2769,8 +2784,8 @@ async function withNotedGroupCore (ctx, key, rec, read) { if (rec === null) return withGroupCore(ctx, key, (core) => read(core, noop)) let stamp = 'none' let skip = false - const value = await withGroupCore(ctx, key, (core) => { - stamp = groupStamp(core) + const value = await withGroupCore(ctx, key, async (core) => { + stamp = await groupStamp(core) return read(core, () => { skip = true }) }) if (!skip) noteDep(rec, 'k', key, stamp) @@ -2964,7 +2979,7 @@ async function readWriterCoreOf (ctx, key, group, rec = null) { if (block !== null) never() return null } - return groupCoreState(core) + return writerCoreState(core) }) } @@ -3110,7 +3125,7 @@ async function foldGroup (ctx, base, group, asPage, kv, rec) { if (core === null) continue const h = core.entries[0].record if (validateWriterHeader(h) !== null || h.group !== group) continue - writers.push({ key, entries: core.entries, diary: h.diary }) + writers.push({ key, entries: core.entries, diary: h.diary, signed: core.signed }) cores.set(key, { diary: h.diary, at: h.at, length: core.length, signed: core.signed, entries: core.entries }) } const diaries = new Map() @@ -5155,13 +5170,14 @@ async function foldCommunity (ctx, base, key, kv, rec) { const core = await readWriterCoreOf(ctx, k, key, rec) if (core === null) continue const h = core.entries[0].record - writers.push({ key: k, entries: core.entries, diary: h.diary }) + writers.push({ key: k, entries: core.entries, diary: h.diary, signed: core.signed }) cores.set(k, { diary: h.diary, at: h.at, length: core.length, signed: core.signed, entries: core.entries }) } const diaries = new Map() const joins = new Map() // The diaries' group-cut marks for C, as the core node reads them: the - // slot walk frees a CW's slot and stops its records at its cut (#350). + // slot walk frees a CW's slot and stops its records at its cut, so a cut + // CW held sparse folds like the whole one (#350). const marks = new Map() const confirms = new Map() for (const d of new Set([ownHex, header.diary, ...writers.map((w) => w.diary)])) { @@ -6960,5 +6976,6 @@ module.exports = { isNeedsExclusive, needsExclusive, lentCore, - entriesStamp + entriesStamp, + replicaStamp } diff --git a/packages/web-member/lib/node-store.js b/packages/web-member/lib/node-store.js index c8dffa2a..4d286b03 100644 --- a/packages/web-member/lib/node-store.js +++ b/packages/web-member/lib/node-store.js @@ -50,6 +50,7 @@ const { const { mediaHeader } = require('@varanda/core/lib/media') const { validateGroupRecord, validateGroupHeader, validateWriterHeader, MAX_GROUPS_PER_DIARY, MAX_GROUP_WRITERS } = require('@varanda/core/lib/group') const { presentState, clearRanges } = require('@varanda/core/lib/deletion') +const { nextHeld } = require('@varanda/core/lib/sparse-held') const { MAX_INVENTORY } = require('@varanda/core/lib/admission') const { SEEN_BLOB_BYTES, SEEN_BLOB_B64 } = require('@varanda/core/lib/seen-blob') const { normalizeBlocks } = require('@varanda/core/lib/member-block') @@ -611,6 +612,30 @@ async function readBlocks (core) { return out } +// A group writer's blocks as readBlocks, then past the first hole only the +// leaves and key blocks held: a seeder clears the rest of a cut writer +// (#350), and the fold reads past a hole only what a cut keeps. +async function readHeldBlocks (core) { + assertOpen(core) + const limit = Math.min(core.length, MAX_LOG_BLOCKS) + const out = [] + let hole = false + for (let i = 0; i < limit; i++) { + const block = (await core.has(i)) ? await core.get(i, { wait: false, raw: true }) : null + if (block === null) { + hole = true + const next = await nextHeld(core, i + 1, limit) + if (next === -1) break + i = next - 1 + continue + } + const record = decodeBlock(block) + if (hole && (record === null || (record.type !== 'group-leave' && record.type !== 'group-key'))) continue + out.push({ seq: i, record }) + } + return out +} + function assertOpen (core) { if (core.closing) { const err = new Error('SESSION_CLOSED: cannot read from a closed session') @@ -2435,6 +2460,7 @@ module.exports = { readRaw, readHeld, readBlocks, + readHeldBlocks, readBlock0, appendRecord, appendRecords, diff --git a/packages/web-member/lib/sync.js b/packages/web-member/lib/sync.js index f2848713..bba7b774 100644 --- a/packages/web-member/lib/sync.js +++ b/packages/web-member/lib/sync.js @@ -189,6 +189,10 @@ // them on its quarantine channel and store none, and a once // run would end with the group behind. While a confirmation // is pending the pages are asked again (GROUP_PUSH_PAGE_MS). +// A writer the own diary cut counts up to the cut, and its +// own leaves and key blocks past it must be advertised by +// that peer, or served to it on the link, too +// (seederHoldsKept). // A seeder from before groups keeps the media rule. // groups the groups of kv 'groupWriters', plus each key of kv // 'groupReplicas' with nothing stored yet (a root asked for @@ -342,7 +346,7 @@ const { attachAdmission, MAX_INVENTORY } = require('@varanda/core/lib/admission' const { createBlocklistState, compareSnapshots } = require('@varanda/core/lib/blocklist') const { attachBlocklist } = require('@varanda/core/lib/blocklist-net') const { attachGroups, GROUPS_LIMIT_MAX } = require('@varanda/core/lib/groups-net') -const { validateGroupHeader, validateWriterHeader, validateGroupRecord, groupJoinHistory, groupConfirmHistory, groupCutHistory, groupJoins, buildCommunity, diaryGroupKeys, slotWalk, MAX_GROUP_WRITERS, MAX_GROUP_WRITERS_PER_DIARY, MAX_FROZEN_WRITERS_PER_DIARY, MAX_WRITER_RELEASES_PER_DIARY, MAX_GROUPS_PER_COMMUNITY } = require('@varanda/core/lib/group') +const { validateGroupHeader, validateWriterHeader, validateGroupRecord, groupJoinHistory, groupConfirmHistory, groupCutHistory, groupJoins, buildCommunity, diaryGroupKeys, diaryWriterCut, slotWalk, MAX_GROUP_WRITERS, MAX_GROUP_WRITERS_PER_DIARY, MAX_FROZEN_WRITERS_PER_DIARY, MAX_WRITER_RELEASES_PER_DIARY, MAX_GROUPS_PER_COMMUNITY } = require('@varanda/core/lib/group') const { presentState, presentLength, @@ -354,6 +358,7 @@ const { foldTargets, isKeptType } = require('@varanda/core/lib/deletion') +const { advertisedHeld, keptPastSeqs, askHeld, peerHolds } = require('@varanda/core/lib/sparse-held') const { memoryStorage } = require('./storage') const { guardedWsStream } = require('./ws-stream') const store = require('./node-store') @@ -428,6 +433,12 @@ const GROUP_PAGE_MS = 30000 // While an own group core waits for its push confirmation, the pages are // asked again at most this often (the channel allows 2 messages a second). const GROUP_PUSH_PAGE_MS = 1000 +// A foreign writer held past its prefix is checked again this often while +// its goal waits on what its peers advertise (core sparse-held.js), and +// less often, up to SPARSE_CHECK_MAX_MS, while nothing changes: a tab runs +// that goal for as long as it is open. +const SPARSE_CHECK_MS = 100 +const SPARSE_CHECK_MAX_MS = 5000 // Writer page requests on one link are spaced this much, all groups and // every re-page together (the seeder refills 2 messages a second). const GROUP_REQUEST_MS = 1000 @@ -603,6 +614,14 @@ function earliestJoin (seq, joins, group, writer) { return out } +// The wait before the next check of a sparse writer's goal: SPARSE_CHECK_MS +// once its held range changed (or first), else twice the last one, up to +// SPARSE_CHECK_MAX_MS. +function sparseCheckDelay (previous, changed) { + if (changed || !(previous > 0)) return SPARSE_CHECK_MS + return Math.min(previous * 2, SPARSE_CHECK_MAX_MS) +} + // Whether an own group core's push is confirmed: the replication peer's // evidence is required (peerHeld), and the page length agrees unless the // seeder does not speak groups (pageLength null). @@ -1648,6 +1667,9 @@ async function syncRun ({ const notCommunities = new Set() // parent keys whose root is a group header: never paged again this run let changedTimer = null const fold = { cursor: 1, seed: null, ops: new Map(), seen: false, running: false, again: false, joinEntries: [], joinCursor: 1, scanPending: false, holes: new Set(), deletedHoles: new Set(), readFails: new Map(), retryTimer: null, holeFilled: false, scanLogged: false } + // The own diary's group-join and group-cut records as of its last scan + // (adoptDiaryGroups), for the cut of each own writer; null before it. + let ownCutEntries = null const diaryWriters = new Set() // writers the own diary binds on:true, seen by adoptDiaryGroups const diaryChase = new Map() // new writer of an adopted root -> its re-page timer let diaryLive = new Map() // writer -> root the own diary binds now, as of the last scan @@ -3488,7 +3510,7 @@ async function syncRun ({ // indexes a tombstone of the own diary deleted (media only; a group core // keeps it empty, seederHoldsMedia reads it for both). function pushEntry (key, core, event) { - const m = { key, core, pushed: -1, stop: null, uploaded: new WeakMap(), event, revoked: new Set() } + const m = { key, core, pushed: -1, stop: null, uploaded: new WeakMap(), event, revoked: new Set(), kept: null, keptReading: false } const onPeer = () => { checkMedia() pollMedia() @@ -3612,13 +3634,63 @@ async function syncRun ({ }) } + // How much of own group core m a seeder must hold: its length, or for a + // writer the own diary cut, up to the cut (block 0 always), since past it + // a seeder keeps only the leaves and key blocks (docs/protocol.md "Sparse + // copies"). + function pushTarget (m, length) { + if (m.key === m.group || ownCutEntries === null) return length + const cut = diaryWriterCut(ownCutEntries, m.group, m.key) + return cut === null ? length : Math.min(length, Math.max(cut, 1)) + } + + // The own leaves and key blocks of writer m past its cut, read once per + // length (null while that read runs; checkMedia runs again after it). + function ownKept (m, cut, length) { + const k = m.kept + if (k !== null && k.from === cut && k.to === length) return k.seqs + if (m.keptReading || ending) return null + m.keptReading = true + const start = k !== null && k.from === cut && k.to < length ? k.to : cut + const before = start === cut ? [] : k.seqs + track(keptPastSeqs(m.core, start, length), (seqs) => { + m.keptReading = false + m.kept = { from: cut, to: length, seqs: before.concat(seqs) } + checkMedia() + }, () => { + m.keptReading = false + }) + return null + } + + // Whether the seeder's peer on link l holds every own leave and key block + // of writer m past the cut the own diary gave it (advertised, or served + // on this link): past the cut a seeder keeps only those, so the writer is + // pushed once it holds them. The peer is asked for its bitfield there (a + // full core sends no wants of its own); a later check sees the answer. + function seederHoldsKept (m, l, length) { + if (m.key === m.group || ownCutEntries === null) return true + const cut = diaryWriterCut(ownCutEntries, m.group, m.key) + if (cut === null || cut >= length) return true + const seqs = ownKept(m, cut, Math.min(length, maxLogBlocks)) + if (seqs === null) return false + if (seqs.length === 0) return true + for (const peer of m.core.peers) { + if (peer.stream !== l.noise) continue + if (peerHolds(peer, seqs, m.uploaded.get(peer) || null)) return true + askHeld(peer, seqs) + return false + } + return false + } + function checkMedia () { const l = seederLink() if (l === null || ended) return for (const m of pushCores()) { const length = m.core.length if (length < 1 || m.pushed === length) continue - if (m.group ? seederHoldsGroup(m, l, length) : seederHoldsMedia(m, l, length)) { + if (m.group ? seederHoldsGroup(m, l, pushTarget(m, length)) && seederHoldsKept(m, l, length) : seederHoldsMedia(m, l, length)) { m.pushed = length emit({ event: m.event, key: m.key, length }) } @@ -3694,7 +3766,11 @@ async function syncRun ({ stopGrow: null, stopFork: null, stopQuota: null, + stopSparse: null, retryTimer: null, + sparse: null, // { length, contiguous, done }: the last advertisedHeld check + sparseTimer: null, + sparseDelay: 0, // the wait before the last check (sparseCheckDelay) reason: null, // why it failed hints: new Set() // 'group:diary:seq' listings whose gate failed on the hint } @@ -4321,9 +4397,10 @@ async function syncRun ({ if (ge.stopGrow) ge.stopGrow() if (ge.stopFork) ge.stopFork() if (ge.stopQuota) ge.stopQuota() + if (ge.stopSparse) ge.stopSparse() if (ge.range) ge.range.destroy() clearLater(ge.retryTimer) - ge.stopGrow = ge.stopFork = ge.stopQuota = ge.range = ge.retryTimer = null + ge.stopGrow = ge.stopFork = ge.stopQuota = ge.stopSparse = ge.range = ge.retryTimer = null } function failGroupEntry (ge, reason, { purge }) { @@ -4416,6 +4493,19 @@ async function syncRun ({ } core.on('append', check) ge.stopQuota = () => core.off('append', check) + // A block past the hole, or a peer coming or going, can meet a sparse + // goal whose check backed off. + const nudge = () => { + if (ge.core === core) sparseNudge(ge) + } + core.on('download', nudge) + core.on('peer-add', nudge) + core.on('peer-remove', nudge) + ge.stopSparse = () => { + core.off('download', nudge) + core.off('peer-add', nudge) + core.off('peer-remove', nudge) + } ge.stopFork = watchFork(core, () => { if (ended || (ge.state !== 'active' && ge.state !== 'regate')) return failGroupEntry(ge, 'fork', { purge: true }) @@ -5297,7 +5387,10 @@ async function syncRun ({ continue } const want = Math.max(ge.core.length, advertised) - if (ge.core.contiguousLength < want) unmet.push(ge.key + ':' + ge.core.contiguousLength + '/' + want) + const ours = ge.core.contiguousLength + // A copy past the advertised prefix with a hole (a writer a seeder keeps + // sparse) is met once its peers' advertised blocks are all held. + if (ours < want && (ours < advertised || !sparseMet(ge))) unmet.push(ge.key + ':' + ours + '/' + want) } // A community's catalogue scan in flight may still add roots to page. for (const [c, st] of catalogueScans) if (st.running || st.again) unmet.push(c + ':catalogue') @@ -5322,6 +5415,42 @@ async function syncRun ({ return unmet } + // Whether foreign writer ge, held up to a hole past what the seeder lists, + // holds every block a connected peer advertises below its signed length, + // from the last check at its present length and prefix (core + // sparse-held.js). A missing or stale check runs again shortly; + // evaluate() follows it. + function sparseMet (ge) { + const s = ge.sparse + if (s && s.length === ge.core.length && s.contiguous === ge.core.contiguousLength && s.done) return true + if (ge.sparseTimer || ended) return false + const changed = !s || s.length !== ge.core.length || s.contiguous !== ge.core.contiguousLength + ge.sparseDelay = sparseCheckDelay(ge.sparseDelay, changed) + ge.sparseTimer = later(() => { + ge.sparseTimer = null + const core = ge.core + if (ended || core === null || ge.state !== 'active') return + const length = core.length + const contiguous = core.contiguousLength + track(advertisedHeld(core, Math.min(length, maxLogBlocks)), (done) => { + if (ge.core !== core) return + ge.sparse = { length, contiguous, done } + evaluate() + }, noop) + }, ge.sparseDelay) + return false + } + + // A check that backed off past SPARSE_CHECK_MS runs again shortly. + function sparseNudge (ge) { + if (!(ge.sparseDelay > SPARSE_CHECK_MS)) return + ge.sparseDelay = 0 + if (!ge.sparseTimer) return + clearLater(ge.sparseTimer) + ge.sparseTimer = null + evaluate() + } + // ---- the follow fold (see the header) ---- // Replays the own diary's follow records from the cursor up to its @@ -5484,6 +5613,7 @@ async function syncRun ({ let added = false const adopted = new Set() const entries = fold.joinEntries.filter((e) => !deleted.has(e.seq)) + ownCutEntries = entries const keys = diaryGroupKeys(entries) for (const key of keys) { if (adoptDiaryRoot(key)) { @@ -5505,6 +5635,8 @@ async function syncRun ({ } const l = seederLink() if (added && l !== null && l.groups !== null) pageGroups(l) + // An own writer the diary cut may be pushed far enough now. + checkMedia() } catch { if (!ended) log('aviso: falha ao ler os grupos do diário') } @@ -5900,4 +6032,4 @@ async function syncRun ({ } } -module.exports = { runSync, retryWait, restartWait, groupVerdict, boxMismatch, addJoins, earliestJoin, groupPushHeld, ownGroupVerdict, bindingRecheck, forgetListed, evictionPurges, capVictim, capExcess, storedJoinProof, refineJoinSeq, MAX_GROUP_HELD_PER_DIARY, WALK_TYPES, liveDiaryWriters, foldedCatalogue, gateVerdict, trackPresent, linearExcept, MAX_RANGES, NEWEST_BLOCKS, TAIL_MAX, HEAD_LOW, HEAD_STALL_MS, GATE_TIMEOUT_MS, CONNECT_TIMEOUT_MS, RETRY_FAST_MAX_MS, RETRY_FAST_WINDOW_MS, RETRY_JITTER, RESTART_WAIT_MS, RESTART_JITTER_MS, MAX_PARALLEL_GATES, MAX_PARALLEL_OPENS, SEEDER_UNAVAILABLE, DIARY_UNAVAILABLE } +module.exports = { runSync, retryWait, sparseCheckDelay, SPARSE_CHECK_MS, SPARSE_CHECK_MAX_MS, restartWait, groupVerdict, boxMismatch, addJoins, earliestJoin, groupPushHeld, ownGroupVerdict, bindingRecheck, forgetListed, evictionPurges, capVictim, capExcess, storedJoinProof, refineJoinSeq, MAX_GROUP_HELD_PER_DIARY, WALK_TYPES, liveDiaryWriters, foldedCatalogue, gateVerdict, trackPresent, linearExcept, MAX_RANGES, NEWEST_BLOCKS, TAIL_MAX, HEAD_LOW, HEAD_STALL_MS, GATE_TIMEOUT_MS, CONNECT_TIMEOUT_MS, RETRY_FAST_MAX_MS, RETRY_FAST_WINDOW_MS, RETRY_JITTER, RESTART_WAIT_MS, RESTART_JITTER_MS, MAX_PARALLEL_GATES, MAX_PARALLEL_OPENS, SEEDER_UNAVAILABLE, DIARY_UNAVAILABLE } diff --git a/packages/web-member/test/group-memo.test.js b/packages/web-member/test/group-memo.test.js index 678df38e..0f73218e 100644 --- a/packages/web-member/test/group-memo.test.js +++ b/packages/web-member/test/group-memo.test.js @@ -15,13 +15,15 @@ const { createGroupKeys } = require('../lib/group-keys') const { createBrowserSession } = require('../lib/session') const ops = require('../lib/member-ops') const store = require('../lib/node-store') +const { groupCutRecord } = require('@varanda/core/lib/records') +const { communityBan } = require('@varanda/core/lib/group') const hex = (buf) => b4a.toString(buf, 'hex') -// Block decodes of group cores (readBlocks) and diary replicas +// Block decodes of group cores (readBlocks, readHeldBlocks) and diary replicas // (readVisible), the work the memo saves. const reads = { n: 0 } -for (const name of ['readBlocks', 'readVisible']) { +for (const name of ['readBlocks', 'readHeldBlocks', 'readVisible']) { const original = store[name] store[name] = (...args) => { reads.n++ @@ -149,6 +151,38 @@ test('the entries stamp follows the seq of a sparse read', (t) => { t.not(entriesStamp([{ seq: 0, record: rec }, { seq: 1, record: null }]), entriesStamp(dense), 'a hole is stamped') }) +// A released stub or a writer cut early holds a few blocks far past its +// prefix (#350): its stamp names them, so the fold memo still holds. +test('a replica holding a few blocks far past its prefix keeps a stamp (#350)', async (t) => { + const Hypercore = require('hypercore') + const RAM = require('random-access-memory') + const source = new Hypercore(RAM) + await source.append(Array.from({ length: 6000 }, (_, i) => b4a.from('b' + i))) + const copy = new Hypercore(RAM, source.key) + await copy.ready() + t.teardown(async () => { + await source.close() + await copy.close() + }) + const a = source.replicate(true, { keepAlive: false }) + const b = copy.replicate(false, { keepAlive: false }) + a.pipe(b).pipe(a) + t.teardown(() => { + a.destroy() + b.destroy() + }) + await copy.download({ start: 0, end: 10 }).done() + await copy.download({ start: 5000, end: 5001 }).done() + t.is(copy.contiguousLength, 10) + t.is(copy.length, 6000) + const first = await ops.replicaStamp(copy) + t.ok(first !== null, 'signed with 5990 blocks missing past its prefix') + t.is(await ops.replicaStamp(copy), first, 'the same copy, the same stamp') + await copy.download({ start: 5500, end: 5501 }).done() + const second = await ops.replicaStamp(copy) + t.ok(second !== null && second !== first, 'a block arriving past the hole changes it') +}) + test('a second group.list decodes no block and answers the same', async (t) => { const { a } = await world(t) const first = await counted(a, () => ops.groupList(a.ctx)) @@ -227,6 +261,73 @@ test('a hide downloaded into a hole of the root invalidates', async (t) => { t.alike(await texts(b, group), ['depois', 'oi'], 'the hide counts') }) +// A seeder keeps a cut writer sparse (#350): below the cut, then its leaves +// and key blocks only. Read past its hole, such a copy folds like the whole +// one, and a leave arriving in that range invalidates. +test('a cut writer held sparse folds like the whole copy, and a leave past its hole invalidates (#350)', async (t) => { + const { a, b, community, group } = await world(t) + await ops.communityJoin(b.ctx, { community }) + await ops.groupJoin(b.ctx, { group }) + await ops.groupPost(b.ctx, { group, text: 'antes' }) + const wB = await ownWriter(b, group) + const cut = await coreLength(b, wB) + await ops.groupPost(b.ctx, { group, text: 'depois' }) + await ops.groupLeave(b.ctx, { group }) + const length = await coreLength(b, wB) + const diary = await store.openDiary(b.storage, b.ctx.keyPair) + try { + await store.appendRecord(diary, groupCutRecord({ group, writer: wB, cut })) + } finally { + await diary.close() + } + await copyDiary(b, a) + await copyCore(b, a, await ownWriter(b, community)) + await copyCore(b, a, wB, [[0, cut]]) + t.alike(await heldRange(a, wB, false), { contiguous: cut, length }, 'the blocks past the cut are not held') + t.alike(await memberKeys(a, group), [a.key, b.key].sort(), 'without the leave, still a member') + t.alike(await texts(a, group), ['antes', 'oi']) + t.is((await counted(a, () => memberKeys(a, group))).n, 0, 'memo warm') + await copyCore(b, a, wB, [[length - 1, length]]) + t.alike(await heldRange(a, wB, false), { contiguous: cut, length }, 'the leave alone past the hole') + const sparse = { members: await memberKeys(a, group), texts: await texts(a, group) } + t.alike(sparse.members, [a.key], 'the leave past the hole counts') + await copyCore(b, a, wB) + t.alike(await heldRange(a, wB, false), { contiguous: length, length }, 'now the whole copy') + t.alike({ members: await memberKeys(a, group), texts: await texts(a, group) }, sparse, 'the sparse copy folded like the whole one') +}) + +test('a cut community writer held sparse folds like the whole copy (#350)', async (t) => { + const { a, b, community } = await world(t) + await ops.communityJoin(b.ctx, { community }) + const cw = await ownWriter(b, community) + const cut = await coreLength(b, cw) + // A record past the cut that the cut drops, so the sparse copy has a hole. + const entry = (await store.readGroupWriters(b.storage)).find((x) => x.group === community) + const core = await store.openGroupWriter(b.storage, await b.ctx.groupKeyPair(new Uint8Array(b4a.from(entry.nonce, 'hex')))) + try { + await store.appendGroupRecord(core, communityBan({ member: a.key, on: true, cuts: [] }), { root: false, community: true }) + } finally { + await core.close() + } + await ops.communityLeave(b.ctx, { community, force: true }) + const length = await coreLength(b, cw) + t.is(length, cut + 2, 'a dropped record, then the leave past the cut') + const diary = await store.openDiary(b.storage, b.ctx.keyPair) + try { + await store.appendRecord(diary, groupCutRecord({ group: community, writer: cw, cut })) + } finally { + await diary.close() + } + const members = async () => (await ops.communityMembers(a.ctx, { community })).members.map((m) => m.member).sort() + await copyDiary(b, a) + await copyCore(b, a, cw, [[0, cut], [length - 1, length]]) + t.alike(await heldRange(a, cw, false), { contiguous: cut, length }, 'the leave alone past the hole') + const sparse = await members() + t.alike(sparse, [a.key], 'the leave past the cut counts') + await copyCore(b, a, cw) + t.alike(await members(), sparse, 'the sparse copy folded like the whole one') +}) + test('a binding diary filling a hole invalidates', async (t) => { const { a, b, community, group } = await world(t) await ops.communityJoin(b.ctx, { community }) diff --git a/packages/web-member/test/group-ops.test.js b/packages/web-member/test/group-ops.test.js index 5611e176..a6330a14 100644 --- a/packages/web-member/test/group-ops.test.js +++ b/packages/web-member/test/group-ops.test.js @@ -914,9 +914,11 @@ test('a group operation reads only that group\'s cores and the diaries binding t await shareGroupCore(b, a, wOther) await shareDiary(b, a) await store.writeFollows(a.storage, [b.ownHex]) - // Group cores are read with readBlocks, diaries with readAll. + // Group roots are read with readBlocks, writers with readHeldBlocks, + // diaries with readAll. const read = realStore.readAll const readBlocks = realStore.readBlocks + const readHeldBlocks = realStore.readHeldBlocks const lengths = [] realStore.readAll = async (core) => { lengths.push(core.length) @@ -926,12 +928,17 @@ test('a group operation reads only that group\'s cores and the diaries binding t lengths.push(core.length) return readBlocks(core) } + realStore.readHeldBlocks = async (core) => { + lengths.push(core.length) + return readHeldBlocks(core) + } let info try { info = await ops.groupInfo(a.ctx, { group }) } finally { realStore.readAll = read realStore.readBlocks = readBlocks + realStore.readHeldBlocks = readHeldBlocks } t.is(info.members, 1, 'B\'s diary binds no core of this group here') // The own diary and this group's root: never B's diary nor the writer diff --git a/packages/web-member/test/sync-groups-sparse.test.js b/packages/web-member/test/sync-groups-sparse.test.js new file mode 100644 index 00000000..da9a17cf --- /dev/null +++ b/packages/web-member/test/sync-groups-sparse.test.js @@ -0,0 +1,184 @@ +// A seeder keeps a writer its diary cut sparse (#350, docs/protocol.md +// "Sparse copies"): below the cut, then only its leaves and key blocks. The +// owner's --once counts that writer pushed once the seeder holds it up to +// its cut and its leaves past it, and a fresh browser member fed by that +// seeder alone ends its --once holding the sparse copy. +const test = require('brittle') +const b4a = require('b4a') +const { testnet, until } = require('./helpers/net') +const { generateAssociationKeyPair } = require('@varanda/core/lib/cert') +const { groupCutRecord } = require('@varanda/core/lib/records') +const grp = require('@varanda/core/lib/group') +const store = require('../lib/node-store') +const ops = require('../lib/member-ops') +const { member, seederOverWs, indexed, sync, once, held } = require('./helpers/sync-groups') + +async function blocksHeld (core, from, to) { + const out = [] + for (let i = from; i < to; i++) if (await core.has(i)) out.push(i) + return out +} + +// -> { length, contiguous, held } of a stored group core. Only with no run +// of m live (helpers/sync-groups.js held()). +async function heldCopy (m, key) { + const core = await store.openGroupCoreReadOnly(m.storage, key) + try { + return { length: core.length, contiguous: core.contiguousLength, held: await blocksHeld(core, 0, core.length) } + } finally { + await core.close() + } +} + +async function ownWriter (m, group) { + const e = (await store.readGroupWriters(m.storage)).find((x) => x.group === group) + return e === undefined ? null : e.key +} + +// A leave on m's own writer in group, written past the member ops as an +// offline device would: group.leave also unbinds the writer in the diary, +// and a seeder lists a writer its diary unbound with that record, which a +// reader's gate refuses (it proves an on: true join). +async function offlineLeave (m, group) { + const entry = (await store.readGroupWriters(m.storage)).find((x) => x.group === group) + const core = await store.openGroupWriter(m.storage, await m.ctx.groupKeyPair(new Uint8Array(b4a.from(entry.nonce, 'hex')))) + try { + await store.appendGroupRecord(core, grp.groupLeave({ seen: null, cuts: [{ writer: entry.key, seq: core.length }] }), {}) + return core.length + } finally { + await core.close() + } +} + +// A tab checks a sparse writer's goal for as long as it is open: the checks +// space out while its held range stays the same, and come back to the +// shortest wait once it changes. +test('a sparse goal is checked less often while nothing changes (#350)', (t) => { + const { sparseCheckDelay, SPARSE_CHECK_MS, SPARSE_CHECK_MAX_MS } = require('../lib/sync') + t.is(sparseCheckDelay(0, false), SPARSE_CHECK_MS, 'the first check') + t.is(sparseCheckDelay(0, true), SPARSE_CHECK_MS) + const waits = [] + let d = 0 + for (let i = 0; i < 12; i++) waits.push((d = sparseCheckDelay(d, i === 0))) + t.alike(waits.slice(0, 4), [SPARSE_CHECK_MS, 2 * SPARSE_CHECK_MS, 4 * SPARSE_CHECK_MS, 8 * SPARSE_CHECK_MS], 'twice the last wait while unchanged') + t.is(waits[waits.length - 1], SPARSE_CHECK_MAX_MS, 'up to the bound') + t.ok(waits.every((w) => w <= SPARSE_CHECK_MAX_MS)) + t.ok(SPARSE_CHECK_MAX_MS <= 5000, 'a goal met is noticed within 5 s') + t.is(sparseCheckDelay(SPARSE_CHECK_MAX_MS, true), SPARSE_CHECK_MS, 'a change brings it back') +}) + +test('a sparse cut writer: the owner\'s and a fresh reader\'s --once end ok (#350)', async (t) => { + t.timeout(240000) + const { bootstrap } = await testnet(t) + const assocKp = generateAssociationKeyPair() + const { seeder, url } = await seederOverWs(t, assocKp, bootstrap) + const a = await member(assocKp, 'Ana') + const b = await member(assocKp, 'Bia') + const r = await member(assocKp, 'Rui') + const { community } = await ops.communityCreate(a.ctx, { name: 'Bairro', join: 'open' }) + const { group } = await ops.groupCreate(a.ctx, { name: 'Horta', join: 'open', community }) + await ops.groupPost(a.ctx, { group, text: 'oi' }) + t.ok((await once(a, url, [])).ok, 'creator pushed') + await until(() => indexed(seeder, group, group), { name: 'root indexed' }) + + await store.addGroupReplica(b.storage, group) + t.ok((await once(b, url, [])).ok, 'B fetched the group and its community') + t.is((await ops.communityJoin(b.ctx, { community })).state, 'member', 'B joined the community') + t.is((await ops.groupJoin(b.ctx, { group })).state, 'member', 'and the group') + await ops.groupPost(b.ctx, { group, text: 'antes' }) + const w = await ownWriter(b, group) + const cut = (await held(b, w)).length + await ops.groupPost(b.ctx, { group, text: 'depois' }) + const length = await offlineLeave(b, group) + const diary = await store.openDiary(b.storage, b.ctx.keyPair) + try { + await store.appendRecord(diary, groupCutRecord({ group, writer: w, cut })) + } finally { + await diary.close() + } + + // B's run pushes its diary and writers; it stops once the seeder holds W + // sparse. + const ac = new AbortController() + const run = sync(b, url, [], { signal: ac.signal }) + try { + await until(async () => { + const e = seeder._groupServed.get(w) + const s = seeder._groups.get(w) + if (!e || !s || e.cutting || s.cut !== cut || s.clearedTo !== length) return false + return (await blocksHeld(e.core, 0, length)).length === cut + 1 + }, { name: 'the seeder holds W sparse', timeoutMs: 60000 }) + } finally { + ac.abort() + await run + } + t.alike(await blocksHeld(seeder._groupServed.get(w).core, 0, length), [...Array(cut).keys(), length - 1], 'the seeder keeps W below its cut and its leave') + + // The seeder holds W up to its cut and its leave: the owner's --once + // counts it pushed. + const rb = await once(b, url, []) + t.ok(rb.ok, 'B\'s run ends ok before its deadline: ' + JSON.stringify(rb.errors)) + + // R, fresh, syncs with the seeder alone; its --once ends once it holds + // every block the seeder advertises. + await store.addGroupReplica(r.storage, group) + const rr = await once(r, url, []) + t.ok(rr.ok, 'R\'s run ends ok before its deadline: ' + JSON.stringify(rr.errors)) + t.alike(await heldCopy(r, w), { length, contiguous: cut, held: [...Array(cut).keys(), length - 1] }, 'R holds W sparse') + const members = (await ops.groupMembers(r.ctx, { group })).members.map((m) => m.member) + t.absent(members.includes(b.ownHex), 'the leave past the hole counts') + const texts = (await ops.groupRead(r.ctx, { group })).items.map((p) => p.text) + t.ok(texts.includes('antes'), 'the post below the cut counts') + t.absent(texts.includes('depois'), 'the post past the cut does not') +}) + +test('the owner confirms a cut writer only once the seeder holds its leave past the cut (#350)', async (t) => { + t.timeout(240000) + const { bootstrap } = await testnet(t) + const assocKp = generateAssociationKeyPair() + const { seeder, url } = await seederOverWs(t, assocKp, bootstrap) + const a = await member(assocKp, 'Ana') + const b = await member(assocKp, 'Bia') + const { community } = await ops.communityCreate(a.ctx, { name: 'Bairro', join: 'open' }) + const { group } = await ops.groupCreate(a.ctx, { name: 'Horta', join: 'open', community }) + await ops.groupPost(a.ctx, { group, text: 'oi' }) + t.ok((await once(a, url, [])).ok, 'creator pushed') + await until(() => indexed(seeder, group, group), { name: 'root indexed' }) + await store.addGroupReplica(b.storage, group) + t.ok((await once(b, url, [])).ok, 'B fetched the group and its community') + await ops.communityJoin(b.ctx, { community }) + await ops.groupJoin(b.ctx, { group }) + await ops.groupPost(b.ctx, { group, text: 'antes' }) + const w = await ownWriter(b, group) + const cut = (await held(b, w)).length + await ops.groupPost(b.ctx, { group, text: 'depois' }) + const length = await offlineLeave(b, group) + const diary = await store.openDiary(b.storage, b.ctx.keyPair) + try { + await store.appendRecord(diary, groupCutRecord({ group, writer: w, cut })) + } finally { + await diary.close() + } + + // The seeder holds W below the cut but has not read past it yet. + const read = seeder._groupPastRange + let resume = null + const stalled = new Promise((resolve) => { resume = resolve }) + seeder._groupPastRange = async function (...args) { + await stalled + return read.apply(this, args) + } + const events = [] + const first = await once(b, url, events, { timeoutMs: 15000 }) + t.absent(first.ok, 'B\'s run does not confirm W at its cut alone') + t.ok(first.errors.join('; ').includes(w + ':push ' + length), 'it waits for W: ' + JSON.stringify(first.errors)) + t.absent(events.some((e) => e.event === 'group-pushed' && e.key === w), 'and never reports it pushed') + t.absent(await seeder._groupServed.get(w).core.has(length - 1), 'control: the seeder lacks the leave') + + resume() + const again = [] + const second = await once(b, url, again) + t.ok(second.ok, 'once the seeder reads past the cut the run ends ok: ' + JSON.stringify(second.errors)) + t.ok(again.some((e) => e.event === 'group-pushed' && e.key === w && e.length === length), 'W reported pushed at its length') + t.ok(await seeder._groupServed.get(w).core.has(length - 1), 'with the leave at the seeder') +})