diff --git a/apps/cli/lib/seed-ws.js b/apps/cli/lib/seed-ws.js index 96a9e353..dd18b314 100644 --- a/apps/cli/lib/seed-ws.js +++ b/apps/cli/lib/seed-ws.js @@ -6,7 +6,7 @@ const path = require('node:path') const { WebSocketServer, createWebSocketStream } = require('ws') const { MAX_EXTERNAL_FRAME } = require('@varanda/core/lib/seeder') const { MEDIA_BLOCK_BYTES } = require('@varanda/core/lib/records') -const { feira } = require('@varanda/rpc') +const { feira, garden: gardenRules } = require('@varanda/rpc') const { renderListingPage, renderNotFound, renderSitemap } = require('./feira-page') const { renderPostPage, postPreviewData, postPhotos } = require('./post-page') @@ -146,17 +146,31 @@ const { renderPostPage, postPreviewData, postPhotos } = require('./post-page') // (as in /index/post) and `next` the same shape or null; any other // segment after the owner is 400 'pedido inválido', never 404, so a // client tells a bad cursor from an unknown owner. +// - /index/garden/ -> Seeder#indexGarden: `{ on: seq | null, state: +// seq | null, helps: [{ author, seq, followSeq }], given: [seq], indexing, +// complete }` (Varandinha; at most GARDEN_HELP_ROWS helps and given): the +// owner's newest live garden block (on or off) and garden-state block, +// the neighbours' garden-help blocks inside the reader window with the +// seq of each helper's live follow of the owner, and the owner's own help +// seqs. /index/garden//rua[/] -> Seeder#indexGardenStreet: +// `{ items: [{ key, on, state }], next, indexing, complete }` (at most +// GARDEN_STREET_PAGE rows), the open balconies of the keys the owner +// follows, key order, strictly after the cursor key; `next` is null or a +// key. Any other segment after the owner is 400 'pedido inválido'. +// Pointers only, checked against the exact shape: an extra key, a +// duplicate or an oversized list is the generic 404, never trimmed. // - /index/profile/ -> Seeder#indexProfile: `{ about: seq | null, // mood: seq | null, fans: { count, indexing }, top: { seq, back: -// { : seq | null } } | null, visits: seq | null, indexing, -// complete }`, the author's newest live profile-about, mood, top-friends -// and visits blocks (perfil rico, melhores amigos, visitas; visits is a -// pointer only, the node peeks the block and checks on), the claimed number of fas and, per listed friend, the +// { : seq | null } } | null, visits: seq | null, garden: seq | +// null, indexing, complete }`, the author's newest live profile-about, +// mood, top-friends, visits and garden blocks (perfil rico, melhores +// amigos, visitas, Varandinha; visits and garden are pointers only, the +// node peeks the block and checks on, so an off block still has a seq), the claimed number of fas and, per listed friend, the // friend's live follow block of the author (at most MAX_TOP_FRIENDS). The // object is open: later features add pointers and readers ignore keys they // do not know, so profileBody keeps only the keys it lists. While the // author's own diary is still being indexed or scanned (fail closed) -// about, mood, top and visits are null, indexing and fans.indexing are true and +// about, mood, top, visits and garden are null, indexing and fans.indexing are true and // complete is false; otherwise complete is true and indexing is the global // flag. Pointers only; the node verifies. // - /index/fans/[/.] -> Seeder#indexFans: `{ author, @@ -420,6 +434,14 @@ const INDEX_SCRAPS_RE = /^\/index\/scraps\/([0-9a-f]{64})(?:\/([0-9]{1,16}\.[0-9 // Any path below an owner's recados that INDEX_SCRAPS_RE refuses: a // malformed cursor (400). const INDEX_SCRAPS_BAD_RE = /^\/index\/scraps\/[0-9a-f]{64}\// +const INDEX_GARDEN_RE = /^\/index\/garden\/([0-9a-f]{64})$/ +const INDEX_GARDEN_STREET_RE = /^\/index\/garden\/([0-9a-f]{64})\/rua(?:\/([0-9a-f]{64}))?$/ +// Any path below an owner's balcony that the two above refuse (400). +const INDEX_GARDEN_BAD_RE = /^\/index\/garden\/[0-9a-f]{64}\// +const GARDEN_KEYS = ['complete', 'given', 'helps', 'indexing', 'on', 'state'] +const GARDEN_HELP_KEYS = ['author', 'followSeq', 'seq'] +const GARDEN_STREET_KEYS = ['complete', 'indexing', 'items', 'next'] +const GARDEN_STREET_ITEM_KEYS = ['key', 'on', 'state'] const INDEX_PROFILE_RE = /^\/index\/profile\/([0-9a-f]{64})$/ const INDEX_FANS_RE = /^\/index\/fans\/([0-9a-f]{64})(?:\/([0-9]{1,16}\.[0-9a-f]{64}))?$/ const FAN_ROWS_MAX = 20 @@ -886,6 +908,14 @@ function startSeedWs ({ host = DEFAULT_HOST, port, seeder, origins = null, media // The atMs of a cursor fits 16 digits but must stay a safe integer. if (m && (m[2] === undefined || Number.isSafeInteger(Number(m[2].split('.')[0])))) return ['indexScraps', [m[1], m[2] === undefined ? null : m[2]], scrapsBody] if (m || INDEX_SCRAPS_BAD_RE.test(url)) return BAD_CURSOR + m = INDEX_GARDEN_STREET_RE.exec(url) + if (m) return ['indexGardenStreet', [m[1], m[2] === undefined ? null : m[2]], gardenStreetBody] + m = INDEX_GARDEN_RE.exec(url) + if (m) { + const owner = m[1] + return ['indexGarden', [owner], (answer) => gardenBody(answer, owner)] + } + if (INDEX_GARDEN_BAD_RE.test(url)) return BAD_CURSOR m = INDEX_PROFILE_RE.exec(url) if (m) return ['indexProfile', [m[1]], profileBody] m = INDEX_FOLLOW_RE.exec(url) @@ -963,11 +993,59 @@ function startSeedWs ({ host = DEFAULT_HOST, port, seeder, origins = null, media } } + // Exactly the Varandinha pointers, checked as the node checks them: a help + // from the owner, a duplicate, an extra key or an oversized list is a 404, + // never a trimmed answer. + function gardenBody (answer, owner) { + if (!exactKeys(answer, GARDEN_KEYS)) throw new Error('index') + const { on, state, helps, given, indexing, complete } = answer + const seqOk = (n) => Number.isSafeInteger(n) && n >= 1 + if ((on !== null && !seqOk(on)) || (state !== null && !seqOk(state))) throw new Error('index') + if (!Array.isArray(helps) || helps.length > gardenRules.GARDEN_HELP_ROWS || !Array.isArray(given) || given.length > gardenRules.GARDEN_HELP_ROWS) throw new Error('index') + const refs = new Set() + const rows = helps.map((row) => { + if (!exactKeys(row, GARDEN_HELP_KEYS)) throw new Error('index') + const { author, seq, followSeq } = row + if (typeof author !== 'string' || !HEX64_RE.test(author) || author === owner || !seqOk(seq) || !seqOk(followSeq)) throw new Error('index') + const ref = author + ':' + seq + if (refs.has(ref)) throw new Error('index') + refs.add(ref) + return { author, seq, followSeq } + }) + const seqs = new Set() + for (const seq of given) { + if (!seqOk(seq) || seqs.has(seq)) throw new Error('index') + seqs.add(seq) + } + if (typeof indexing !== 'boolean' || typeof complete !== 'boolean') throw new Error('index') + return { on, state, helps: rows, given: [...given], indexing, complete } + } + + // Exactly one page of Varandas da rua: a duplicate key, an extra key or an + // oversized page is a 404. + function gardenStreetBody (answer) { + if (!exactKeys(answer, GARDEN_STREET_KEYS)) throw new Error('index') + const { items, next, indexing, complete } = answer + const seqOk = (n) => Number.isSafeInteger(n) && n >= 1 + if (!Array.isArray(items) || items.length > gardenRules.GARDEN_STREET_PAGE) throw new Error('index') + const keys = new Set() + const rows = items.map((item) => { + if (!exactKeys(item, GARDEN_STREET_ITEM_KEYS)) throw new Error('index') + const { key, on, state } = item + if (typeof key !== 'string' || !HEX64_RE.test(key) || keys.has(key) || !seqOk(on) || (state !== null && !seqOk(state))) throw new Error('index') + keys.add(key) + return { key, on, state } + }) + if (next !== null && (typeof next !== 'string' || !HEX64_RE.test(next))) throw new Error('index') + if (typeof indexing !== 'boolean' || typeof complete !== 'boolean') throw new Error('index') + return { items: rows, next, indexing, complete } + } + // The pointer keys known so far; a new key is added here when its feature // ships, an unknown one is never echoed. - function profileBody ({ about, mood, fans = null, top = null, visits = null, indexing, complete }) { + function profileBody ({ about, mood, fans = null, top = null, visits = null, garden = null, indexing, complete }) { const seqOk = (n) => n === null || (Number.isSafeInteger(n) && n >= 1) - if (!seqOk(about) || !seqOk(mood) || !seqOk(visits)) throw new Error('index') + if (!seqOk(about) || !seqOk(mood) || !seqOk(visits) || !seqOk(garden)) throw new Error('index') let fanCount = null if (fans !== null) { if (typeof fans !== 'object' || !Number.isSafeInteger(fans.count) || fans.count < 0) throw new Error('index') @@ -985,7 +1063,7 @@ function startSeedWs ({ host = DEFAULT_HOST, port, seeder, origins = null, media } pointers = { seq: top.seq, back } } - return { about, mood, fans: fanCount, top: pointers, visits, indexing: indexing === true, complete: complete === true } + return { about, mood, fans: fanCount, top: pointers, visits, garden, indexing: indexing === true, complete: complete === true } } // Only the follower-list fields, checked as the client checks them: a @@ -2092,6 +2170,8 @@ module.exports = { INDEX_FOLLOWS_RE, INDEX_TESTIMONIALS_RE, INDEX_SCRAPS_RE, + INDEX_GARDEN_RE, + INDEX_GARDEN_STREET_RE, INDEX_PROFILE_RE, INDEX_FANS_RE, INDEX_POLL_RE, diff --git a/apps/cli/test/seed-ws.test.js b/apps/cli/test/seed-ws.test.js index acf8f695..ac0f5a9c 100644 --- a/apps/cli/test/seed-ws.test.js +++ b/apps/cli/test/seed-ws.test.js @@ -3367,6 +3367,184 @@ test('startSeedWs /index/scraps and /index/follow: malformed answers are 404, th t.is(busy.headers['retry-after'], '1') }) +// /index/garden/ and /index/garden//rua[/]: Varandinha +// pointers, checked against the exact shape; a bad tail is 400. +const GARDEN_E = 'e'.repeat(64) +const gardenOf = () => ({ on: 3, state: 9, helps: [{ author: B, seq: 4, followSeq: 2 }, { author: GARDEN_E, seq: 1, followSeq: 6 }], given: [12, 8], indexing: false, complete: true }) +const gardenStreetOf = () => ({ items: [{ key: B, on: 2, state: null }, { key: GARDEN_E, on: 5, state: 7 }], next: GARDEN_E, indexing: true, complete: false }) + +function gardenSeeder ({ garden = gardenOf, street = gardenStreetOf } = {}) { + const stub = indexSeeder() + stub.indexGarden = async (owner) => { + stub.calls.push(['indexGarden', owner]) + return owner === A ? garden() : null + } + stub.indexGardenStreet = async (owner, cursor) => { + stub.calls.push(['indexGardenStreet', owner, cursor]) + return owner === A ? street() : null + } + return stub +} + +test('startSeedWs /index/garden: regexes, pointers, the street with a cursor, HEAD, 404, 400 for a bad tail, 421 and 405', async (t) => { + t.timeout(20000) + const { INDEX_GARDEN_RE, INDEX_GARDEN_STREET_RE } = require('../lib/seed-ws') + t.is(INDEX_GARDEN_RE.source, '^\\/index\\/garden\\/([0-9a-f]{64})$') + t.is(INDEX_GARDEN_STREET_RE.source, '^\\/index\\/garden\\/([0-9a-f]{64})\\/rua(?:\\/([0-9a-f]{64}))?$') + const seeder = gardenSeeder() + const server = await startSeedWs({ port: 0, seeder }) + t.teardown(() => server.close(), { force: true }) + const url = '/index/garden/' + A + const got = await mediaRequest(server.port, url) + t.is(got.status, 200) + jsonHeaders(t, got) + t.alike(JSON.parse(got.body.toString()), gardenOf()) + t.alike(seeder.calls.at(-1), ['indexGarden', A]) + const head = await mediaRequest(server.port, url, { method: 'HEAD' }) + t.is(head.status, 200) + t.is(head.headers['content-length'], got.headers['content-length']) + t.is(head.body.byteLength, 0) + t.is((await mediaRequest(server.port, '/index/garden/' + 'f'.repeat(64))).status, 404, 'an unknown owner') + + const street = await mediaRequest(server.port, url + '/rua') + t.is(street.status, 200) + jsonHeaders(t, street) + t.alike(JSON.parse(street.body.toString()), gardenStreetOf()) + t.alike(seeder.calls.at(-1), ['indexGardenStreet', A, null]) + t.is((await mediaRequest(server.port, url + '/rua/' + B)).status, 200) + t.alike(seeder.calls.at(-1), ['indexGardenStreet', A, B], 'the cursor reaches the seeder as given') + t.is((await mediaRequest(server.port, '/index/garden/' + 'f'.repeat(64) + '/rua')).status, 404, 'an unknown owner on the street') + + seeder.calls.length = 0 + const notFound = ['/index/garden', '/index/garden/', '/index/garden/' + 'a'.repeat(63), '/index/garden/' + 'A'.repeat(64), '/index/garden/' + 'a'.repeat(65), + url + '?x=1', '/index/garden/' + A.toUpperCase() + '/rua'] + for (const u of notFound) { + const r = await mediaRequest(server.port, u) + t.is(r.status, 404, u + ' -> 404') + t.is(r.body.toString(), 'não encontrado\n') + } + const badTail = ['/', '/x', '/rua/', '/rua?x=1', '/Rua', '/ruas', '/rua/' + B.toUpperCase(), '/rua/' + 'b'.repeat(63), '/rua/' + 'b'.repeat(65), + '/rua/' + B + '/', '/rua/' + B + '/x', '/rua/' + CURSOR, '/' + B] + for (const suffix of badTail) { + const r = await mediaRequest(server.port, url + suffix) + t.is(r.status, 400, suffix + ' -> 400') + t.is(r.body.toString(), 'pedido inválido\n') + t.is(r.headers['cache-control'], 'no-store') + } + t.is(seeder.calls.length, 0, 'malformed paths never reach the seeder') + t.is((await mediaRequest(server.port, url, { host: 'evil.example:' + server.port })).status, 421) + t.is((await mediaRequest(server.port, url + '/rua', { host: 'evil.example:' + server.port })).status, 421) + for (const method of ['POST', 'PUT', 'DELETE', 'PATCH']) { + for (const u of [url, url + '/rua', url + '/rua/' + B, url + '/x']) { + const r = await mediaRequest(server.port, u, { method }) + t.is(r.status, 405, method + ' ' + u + ' -> 405') + t.is(r.headers.allow, 'GET, HEAD') + } + } + t.is(seeder.calls.length, 0, 'refused requests never reach the seeder') +}) + +test('startSeedWs /index/garden: an answer off the exact shape is the generic 404, the worst case fits, both spend the index bucket', async (t) => { + t.timeout(30000) + const hex = (i) => i.toString(16).padStart(64, '0') + const big = Number.MAX_SAFE_INTEGER + let garden = null + let street = null + const seeder = indexSeeder() + const server = await startSeedWs({ port: 0, seeder }) + t.teardown(() => server.close(), { force: true }) + const url = '/index/garden/' + A + t.is((await mediaRequest(server.port, url)).status, 404, 'a seeder without the method') + t.is((await mediaRequest(server.port, url + '/rua')).status, 404, 'a seeder without the method') + seeder.indexGarden = async () => garden + seeder.indexGardenStreet = async () => street + + const worstGarden = { on: big, state: big, helps: Array.from({ length: 24 }, (_, i) => ({ author: hex(i), seq: big - i, followSeq: big })), given: Array.from({ length: 24 }, (_, i) => big - i), indexing: true, complete: false } + garden = worstGarden + const wg = await mediaRequest(server.port, url) + t.is(wg.status, 200) + t.ok(wg.body.byteLength <= INDEX_MAX_BODY, 'garden worst case is ' + wg.body.byteLength + ' bytes') + t.alike(JSON.parse(wg.body.toString()), worstGarden) + garden = { on: null, state: null, helps: [], given: [], indexing: true, complete: false } + const pending = await mediaRequest(server.port, url) + t.is(pending.status, 200, 'the pending answer passes') + t.alike(JSON.parse(pending.body.toString()), garden) + + const worstStreet = { items: Array.from({ length: 20 }, (_, i) => ({ key: hex(i), on: big, state: big })), next: 'f'.repeat(64), indexing: true, complete: false } + street = worstStreet + const ws = await mediaRequest(server.port, url + '/rua') + t.is(ws.status, 200) + t.ok(ws.body.byteLength <= INDEX_MAX_BODY, 'street worst case is ' + ws.body.byteLength + ' bytes') + t.alike(JSON.parse(ws.body.toString()), worstStreet) + + const help = { author: B, seq: 4, followSeq: 2 } + const gardenCases = [ + ['an extra key', { ...gardenOf(), secret: 'x' }], + ['a missing key', (() => { const o = gardenOf(); delete o.given; return o })()], + ['an extra help key', { ...gardenOf(), helps: [{ ...help, at: '2026-01-01T00:00:00.000Z' }] }], + ['a help without followSeq', { ...gardenOf(), helps: [{ author: B, seq: 4 }] }], + ['a help with a null followSeq', { ...gardenOf(), helps: [{ ...help, followSeq: null }] }], + ['a help from the owner', { ...gardenOf(), helps: [{ ...help, author: A }] }], + ['a duplicate help', { ...gardenOf(), helps: [help, { ...help }] }], + ['a help with seq 0', { ...gardenOf(), helps: [{ ...help, seq: 0 }] }], + ['a help with a string seq', { ...gardenOf(), helps: [{ ...help, seq: '4' }] }], + ['a help with an uppercase author', { ...gardenOf(), helps: [{ ...help, author: B.toUpperCase() }] }], + ['a help that is an array', { ...gardenOf(), helps: [[B, 4, 2]] }], + ['25 helps', { ...gardenOf(), helps: [...worstGarden.helps, { author: hex(99), seq: 1, followSeq: 1 }] }], + ['helps not an array', { ...gardenOf(), helps: { 0: help } }], + ['25 given', { ...gardenOf(), given: [...worstGarden.given, 1] }], + ['a duplicate given', { ...gardenOf(), given: [8, 8] }], + ['a given of 0', { ...gardenOf(), given: [0] }], + ['a fractional given', { ...gardenOf(), given: [1.5] }], + ['on 0', { ...gardenOf(), on: 0 }], + ['on true', { ...gardenOf(), on: true }], + ['a string state', { ...gardenOf(), state: '9' }], + ['an unsafe state', { ...gardenOf(), state: 2 ** 53 }], + ['a non-boolean indexing', { ...gardenOf(), indexing: 0 }], + ['a non-boolean complete', { ...gardenOf(), complete: 'true' }] + ] + for (const [label, value] of gardenCases) { + garden = value + const r = await mediaRequest(server.port, url) + t.is(r.status, 404, 'garden: ' + label + ' -> 404') + t.is(r.body.toString(), 'não encontrado\n') + } + const item = { key: B, on: 2, state: null } + const streetCases = [ + ['an extra key', { ...gardenStreetOf(), secret: 'x' }], + ['a rows key', (() => { const o = gardenStreetOf(); o.rows = o.items; delete o.items; return o })()], + ['an extra item key', { ...gardenStreetOf(), items: [{ ...item, name: 'segredo' }] }], + ['an item without state', { ...gardenStreetOf(), items: [{ key: B, on: 2 }] }], + ['an item with a null on', { ...gardenStreetOf(), items: [{ ...item, on: null }] }], + ['an item with state 0', { ...gardenStreetOf(), items: [{ ...item, state: 0 }] }], + ['a duplicate key', { ...gardenStreetOf(), items: [item, { ...item, on: 3 }] }], + ['an uppercase key', { ...gardenStreetOf(), items: [{ ...item, key: B.toUpperCase() }] }], + ['21 items', { ...gardenStreetOf(), items: [...worstStreet.items, { key: hex(99), on: 1, state: null }] }], + ['items not an array', { ...gardenStreetOf(), items: null }], + ['a malformed next', { ...gardenStreetOf(), next: CURSOR }], + ['a numeric next', { ...gardenStreetOf(), next: 7 }], + ['a non-boolean complete', { ...gardenStreetOf(), complete: 1 }] + ] + for (const [label, value] of streetCases) { + street = value + const r = await mediaRequest(server.port, url + '/rua') + t.is(r.status, 404, 'street: ' + label + ' -> 404') + t.is(r.body.toString(), 'não encontrado\n') + } + seeder.indexGarden = async () => { throw new Error('segredo') } + const thrown = await mediaRequest(server.port, url) + t.is(thrown.status, 404) + t.absent(thrown.body.toString().includes('segredo')) + + const s2 = await startSeedWs({ port: 0, seeder: gardenSeeder(), index: { rate: 0, burst: 2 } }) + t.teardown(() => s2.close(), { force: true }) + t.is((await mediaRequest(s2.port, url)).status, 200, 'garden spends a token') + t.is((await mediaRequest(s2.port, url + '/rua')).status, 200, 'the street spends a token') + const busy = await mediaRequest(s2.port, '/index/member/' + A) + t.is(busy.status, 503, 'the shared index bucket is empty') + t.is(busy.headers['retry-after'], '1') +}) + // /index/profile/: the perfil rico pointers, rebuilt key by key. test('startSeedWs /index/profile: regex, pointers only, HEAD, 404, 421, 405 and the index bucket', async (t) => { t.timeout(20000) @@ -3384,7 +3562,7 @@ test('startSeedWs /index/profile: regex, pointers only, HEAD, 404, 421, 405 and const got = await mediaRequest(server.port, url) t.is(got.status, 200) jsonHeaders(t, got) - t.alike(JSON.parse(got.body.toString()), { about: 3, mood: null, fans: null, top: null, visits: null, indexing: false, complete: false }, 'only the known keys leave') + t.alike(JSON.parse(got.body.toString()), { about: 3, mood: null, fans: null, top: null, visits: null, garden: null, indexing: false, complete: false }, 'only the known keys leave') t.alike(seeder.calls.at(-1), ['indexProfile', A]) const head = await mediaRequest(server.port, url, { method: 'HEAD' }) t.is(head.status, 200) @@ -3415,7 +3593,7 @@ test('startSeedWs /index/profile: a malformed answer, a throw or a missing metho seeder.indexProfile = async () => ({ about: big, mood: big, indexing: true }) const worst = await mediaRequest(server.port, url) t.is(worst.status, 200) - t.alike(JSON.parse(worst.body.toString()), { about: big, mood: big, fans: null, top: null, visits: null, indexing: true, complete: false }) + t.alike(JSON.parse(worst.body.toString()), { about: big, mood: big, fans: null, top: null, visits: null, garden: null, indexing: true, complete: false }) for (const answer of [{ about: 0, mood: null, indexing: false }, { about: null, mood: 1.5, indexing: false }, { about: '3', mood: null, indexing: false }, { about: undefined, mood: null, indexing: false }, { about: -1, mood: null, indexing: false }]) { seeder.indexProfile = async () => answer @@ -3443,13 +3621,13 @@ test('startSeedWs /index/profile: fans and top are rebuilt key by key; a malform const server = await startSeedWs({ port: 0, seeder }) t.teardown(() => server.close(), { force: true }) const url = '/index/profile/' + A - seeder.indexProfile = async () => ({ about: 1, mood: null, fans: { count: 2, indexing: true, extra: 'x' }, top: { seq: 4, back: { [B]: 7, [hex(3)]: null }, keys: ['segredo'] }, visits: 6, indexing: false, complete: true }) + seeder.indexProfile = async () => ({ about: 1, mood: null, fans: { count: 2, indexing: true, extra: 'x' }, top: { seq: 4, back: { [B]: 7, [hex(3)]: null }, keys: ['segredo'] }, visits: 6, garden: 5, indexing: false, complete: true }) const got = await mediaRequest(server.port, url) t.is(got.status, 200) - t.alike(JSON.parse(got.body.toString()), { about: 1, mood: null, fans: { count: 2, indexing: true }, top: { seq: 4, back: { [B]: 7, [hex(3)]: null } }, visits: 6, indexing: false, complete: true }, 'only the known keys leave') + t.alike(JSON.parse(got.body.toString()), { about: 1, mood: null, fans: { count: 2, indexing: true }, top: { seq: 4, back: { [B]: 7, [hex(3)]: null } }, visits: 6, garden: 5, indexing: false, complete: true }, 'only the known keys leave') const back = {} for (let i = 0; i < 8; i++) back[hex(i)] = big - const worst = { about: big, mood: big, fans: { count: big, indexing: true }, top: { seq: big, back }, visits: big, indexing: true, complete: false } + const worst = { about: big, mood: big, fans: { count: big, indexing: true }, top: { seq: big, back }, visits: big, garden: big, indexing: true, complete: false } seeder.indexProfile = async () => worst const w = await mediaRequest(server.port, url) t.is(w.status, 200) @@ -3460,6 +3638,7 @@ test('startSeedWs /index/profile: fans and top are rebuilt key by key; a malform { fans: { count: -1, indexing: false } }, { fans: { count: 1.5, indexing: false } }, { fans: { count: '2', indexing: false } }, { fans: 3 }, { top: { seq: 0, back: {} } }, { top: { seq: 1 } }, { top: { seq: 1, back: null } }, { top: { seq: 1, back: { x: 1 } } }, { visits: 0 }, { visits: 1.5 }, { visits: '3' }, { visits: -1 }, + { garden: 0 }, { garden: 1.5 }, { garden: '3' }, { garden: -1 }, { garden: true }, { top: { seq: 1, back: { [B]: 0 } } }, { top: { seq: 1, back: { [B]: '3' } } }, { top: { seq: 1, back: { [B.toUpperCase()]: 1 } } }, { top: { seq: 1, back: nine } }, { top: 'x' } ]) { seeder.indexProfile = async () => ({ about: null, mood: null, indexing: false, complete: true, ...bad }) diff --git a/apps/cli/test/seeder-index.test.js b/apps/cli/test/seeder-index.test.js index 0b6f367f..e318c130 100644 --- a/apps/cli/test/seeder-index.test.js +++ b/apps/cli/test/seeder-index.test.js @@ -2240,8 +2240,8 @@ test('indexProfile: newest about and mood pointers, deletes, fail closed while i await pass() } await pass() - t.alike(seeder.indexProfile(O), { about: 1, mood: 2, fans: { count: 0, indexing: false }, top: null, visits: null, indexing: false, complete: true }) - t.alike(seeder.indexProfile(V), { about: null, mood: 1, fans: { count: 0, indexing: false }, top: null, visits: null, indexing: false, complete: true }, 'each author has its own pointers') + t.alike(seeder.indexProfile(O), { about: 1, mood: 2, fans: { count: 0, indexing: false }, top: null, visits: null, garden: null, indexing: false, complete: true }) + t.alike(seeder.indexProfile(V), { about: null, mood: 1, fans: { count: 0, indexing: false }, top: null, visits: null, garden: null, indexing: false, complete: true }, 'each author has its own pointers') t.is(seeder.indexProfile('f'.repeat(64)), null, 'an author not served is unknown') t.is(seeder.indexProfile('nope'), null) @@ -2250,19 +2250,19 @@ test('indexProfile: newest about and mood pointers, deletes, fail closed while i await grow(O, ofO, [deleteRecord({ seq: 3, at: at(4) })]) // 4 t.is(seeder.indexProfile(O).about, 1, 'a deleted block falls back to the previous one') await grow(O, ofO, [deleteRecord({ seq: 2, at: at(5) })]) // 5 - t.alike(seeder.indexProfile(O), { about: 1, mood: null, fans: { count: 0, indexing: false }, top: null, visits: null, indexing: false, complete: true }) + t.alike(seeder.indexProfile(O), { about: 1, mood: null, fans: { count: 0, indexing: false }, top: null, visits: null, garden: null, indexing: false, complete: true }) seeder._rebuildIndex() - t.alike(seeder.indexProfile(O), { about: null, mood: null, fans: { count: 0, indexing: true }, top: null, visits: null, indexing: true, complete: false }, 'between a rebuild and the re-index: fail closed') + t.alike(seeder.indexProfile(O), { about: null, mood: null, fans: { count: 0, indexing: true }, top: null, visits: null, garden: null, indexing: true, complete: false }, 'between a rebuild and the re-index: fail closed') await settleIndex(seeder) - t.alike(seeder.indexProfile(O), { about: 1, mood: null, fans: { count: 0, indexing: false }, top: null, visits: null, indexing: false, complete: true }, 'the same after a rebuild') + t.alike(seeder.indexProfile(O), { about: 1, mood: null, fans: { count: 0, indexing: false }, top: null, visits: null, garden: null, indexing: false, complete: true }, 'the same after a rebuild') const owner = seeder._served.get(O) owner.indexPaused = true - t.alike(seeder.indexProfile(O), { about: null, mood: null, fans: { count: 0, indexing: true }, top: null, visits: null, indexing: true, complete: false }, 'a paused diary: no pointer') + t.alike(seeder.indexProfile(O), { about: null, mood: null, fans: { count: 0, indexing: true }, top: null, visits: null, garden: null, indexing: true, complete: false }, 'a paused diary: no pointer') owner.indexPaused = false owner.core.contiguousLength++ - t.alike(seeder.indexProfile(O), { about: null, mood: null, fans: { count: 0, indexing: true }, top: null, visits: null, indexing: true, complete: false }, 'a diary behind its present length: no pointer') + t.alike(seeder.indexProfile(O), { about: null, mood: null, fans: { count: 0, indexing: true }, top: null, visits: null, garden: null, indexing: true, complete: false }, 'a diary behind its present length: no pointer') owner.core.contiguousLength-- t.is(seeder.indexProfile(O).about, 1, 'caught up: the pointer again') @@ -2362,6 +2362,7 @@ test('indexProfile and indexFans: fan count and pages, top-friends with follow-b fans: { count: 2, indexing: false }, top: { seq: 1, back: { [F]: 2, [G]: null, [H]: null } }, visits: null, + garden: null, indexing: false, complete: true }, 'two fas (H took it back), F follows back, G unfollowed, H never followed') @@ -2640,6 +2641,235 @@ test('indexEvents: event post pointers, cursor, fail closed, held, blocked, dele t.is(seeder.indexEvents(O), null, 'a purged author is unknown') }) +// Garden helps are listed only inside the reader window of Date.now(), so +// these records are dated relative to the real clock, not BASE_MS. +function gardenFixture (all) { + const { seeder, diary } = scanFixture() + const entries = new Map(all.map(([key, records]) => [key, scannable(seeder, diary(key, records.length, (seq) => records[seq - 1]))])) + const pass = async () => { + for (const [key] of all) { + const entry = entries.get(key) + if (!entry || entry.dropped) continue + await seeder._runScan(key, entry) + seeder._onGrow(key, entry, entry.core.contiguousLength) + } + await settleIndex(seeder) + } + const grow = async (key, records, more) => { + records.push(...more) + entries.get(key).core.contiguousLength = records.length + 1 + await pass() + } + return { seeder, pass, grow } +} + +const ISO_DATE = /\d{4}-\d{2}-\d{2}T/ + +test('indexGarden: on, state, helps from followers with their follow pointer, given, deletes, fail closed, held, blocked and purged', async (t) => { + const { gardenRecord, gardenStateRecord, gardenHelpRecord, followRecord } = require('@varanda/core/lib/records') + const A = authorKey(1) + const B = authorKey(2) + const C = authorKey(3) + const D = authorKey(4) + const ago = (h) => new Date(Date.now() - h * 3600000).toISOString() + const pot = { plant: 'manjericao', at: ago(5), water: ago(5), w: 0 } + // A shows a balcony with one pot and watered B's; B and D follow A and + // water it; C waters it without following A. + const ofA = [gardenRecord({ on: true, at: ago(5) }), gardenStateRecord({ pots: [pot, null, null, null, null, null], n: 1, at: ago(5) }), gardenHelpRecord({ to: B, pot: 0, at: ago(4) })] + const ofB = [followRecord({ key: A, on: true, at: at(0) }), gardenHelpRecord({ to: A, pot: 0, at: ago(3) })] + const ofC = [gardenHelpRecord({ to: A, pot: 0, at: ago(2) })] + const ofD = [followRecord({ key: A, on: true, at: at(1) }), gardenHelpRecord({ to: A, pot: 0, at: ago(1) })] + const { seeder, pass, grow } = gardenFixture([[A, ofA], [B, ofB], [C, ofC], [D, ofD]]) + await pass() + const full = { + on: 1, + state: 2, + helps: [{ author: D, seq: 2, followSeq: 1 }, { author: B, seq: 2, followSeq: 1 }], + given: [3], + indexing: false, + complete: true + } + t.alike(seeder.indexGarden(A), full, 'followers\' helps newest first, each with the helper\'s follow of A; C does not follow A and is left out') + const text = JSON.stringify(seeder.indexGarden(A)) + t.absent(text.includes('manjericao') || text.includes('pot') || ISO_DATE.test(text), 'pointers only: no pot, plant or date') + t.alike(seeder.indexGarden(B), { on: null, state: null, helps: [], given: [], indexing: false, complete: true }, 'a member without a balcony: null pointers, nothing listed') + t.is(seeder.indexGarden('f'.repeat(64)), null, 'an owner not served is unknown, never an empty balcony') + t.is(seeder.indexGarden('nope'), null) + + await grow(C, ofC, [followRecord({ key: A, on: true, at: at(2) })]) // C:2 + t.alike(seeder.indexGarden(A).helps, [{ author: D, seq: 2, followSeq: 1 }, { author: C, seq: 1, followSeq: 2 }, { author: B, seq: 2, followSeq: 1 }], 'once C follows A its help appears') + await grow(C, ofC, [followRecord({ key: A, on: false, at: at(3) })]) // C:3 + t.alike(seeder.indexGarden(A), full, 'an unfollow takes it out again') + + await grow(D, ofD, [deleteRecord({ seq: 2, at: at(4) })]) // D:3 + t.alike(seeder.indexGarden(A).helps, [{ author: B, seq: 2, followSeq: 1 }], 'a deleted help is gone') + await grow(A, ofA, [gardenRecord({ on: false, at: ago(0.5) })]) // A:4 + t.alike(seeder.indexGarden(A), { on: 4, state: null, helps: [], given: [], indexing: false, complete: true }, 'an off flag is still the pointer, with no state, helps or given') + await grow(A, ofA, [deleteRecord({ seq: 4, at: at(5) })]) // A:5 + const kept = { ...full, helps: [{ author: B, seq: 2, followSeq: 1 }] } + t.alike(seeder.indexGarden(A), kept, 'a deleted flag falls back to the previous one') + + const pending = { on: null, state: null, helps: [], given: [], indexing: true, complete: false } + seeder._rebuildIndex() + t.alike(seeder.indexGarden(A), pending, 'between a rebuild and the re-index: fail closed') + await settleIndex(seeder) + t.alike(seeder.indexGarden(A), kept, 'the same after a rebuild') + + const owner = seeder._served.get(A) + owner.indexPaused = true + t.alike(seeder.indexGarden(A), pending, 'a paused owner: nothing listed') + owner.indexPaused = false + owner.scanned = 0 + t.alike(seeder.indexGarden(A), pending, 'a tombstone scan behind: nothing listed') + owner.scanned = owner.present + t.alike(seeder.indexGarden(A), kept, 'caught up') + + seeder._served.get(B).searchHold = 1 + t.alike(seeder.indexGarden(A), { ...kept, helps: [], complete: false }, 'a held helper is left out and the answer is not complete') + seeder._served.get(B).searchHold = 0 + seeder._isBlocked = (hex) => hex === B + t.alike(seeder.indexGarden(A), { ...kept, helps: [], complete: false }, 'a blocked helper is left out and the answer is not complete') + seeder._isBlocked = (hex) => hex === A + t.is(seeder.indexGarden(A), null, 'a blocked owner is unknown') + seeder._isBlocked = () => false + owner.searchHold = 1 + t.is(seeder.indexGarden(A), null, 'a held owner is unknown') + owner.searchHold = 0 + + seeder._drop(B, 'blocked') + t.alike(seeder.indexGarden(A), { ...kept, helps: [] }, 'a purged helper leaves at once') + seeder._drop(A, 'blocked') + t.is(seeder.indexGarden(A), null, 'a purged owner is unknown') +}) + +test('indexGardenStreet: only followed balconies that are on, cursor, unfollow, fail closed, held, blocked and purged', async (t) => { + const { gardenRecord, gardenStateRecord, followRecord } = require('@varanda/core/lib/records') + const O = authorKey(1) + const A = authorKey(2) + const B = authorKey(3) + const C = authorKey(4) + const D = authorKey(5) + const ago = (h) => new Date(Date.now() - h * 3600000).toISOString() + // O follows A, B and C; A and B show their balcony, C turned it off; D + // shows one but O does not follow D. + const ofO = [followRecord({ key: A, on: true, at: at(0) }), followRecord({ key: B, on: true, at: at(1) }), followRecord({ key: C, on: true, at: at(2) })] + const ofA = [gardenRecord({ on: true, at: ago(3) }), gardenStateRecord({ pots: [null, null, null, null, null, null], n: 1, at: ago(3) })] + const ofB = [gardenRecord({ on: true, at: ago(2) })] + const ofC = [gardenRecord({ on: true, at: ago(2) }), gardenRecord({ on: false, at: ago(1) })] + const ofD = [gardenRecord({ on: true, at: ago(1) })] + const { seeder, pass, grow } = gardenFixture([[O, ofO], [A, ofA], [B, ofB], [C, ofC], [D, ofD]]) + await pass() + const full = { items: [{ key: A, on: 1, state: 2 }, { key: B, on: 1, state: null }], next: null, indexing: false, complete: true } + t.alike(seeder.indexGardenStreet(O), full, 'followed keys that are on, keys ascending; C is off and D is not followed') + t.absent(ISO_DATE.test(JSON.stringify(seeder.indexGardenStreet(O))), 'keys and seqs only') + t.alike(seeder.indexGardenStreet(O, A), { ...full, items: [{ key: B, on: 1, state: null }] }, 'an exclusive cursor') + t.alike(seeder.indexGardenStreet(D), { items: [], next: null, indexing: false, complete: true }, 'an owner who follows nobody') + t.is(seeder.indexGardenStreet(O, 'x'), null, 'a malformed cursor') + t.is(seeder.indexGardenStreet('f'.repeat(64)), null, 'an owner not served is unknown') + t.is(seeder.indexGardenStreet('nope'), null) + + await grow(B, ofB, [gardenRecord({ on: false, at: ago(0.5) })]) // B:2 + t.alike(seeder.indexGardenStreet(O).items, [{ key: A, on: 1, state: 2 }], 'a balcony turned off leaves the street') + await grow(B, ofB, [deleteRecord({ seq: 2, at: at(3) })]) // B:3 + t.alike(seeder.indexGardenStreet(O), full, 'a deleted off flag falls back to on') + + seeder._rebuildIndex() + t.alike(seeder.indexGardenStreet(O), { items: [], next: null, indexing: true, complete: false }, 'between a rebuild and the re-index: fail closed') + await settleIndex(seeder) + t.alike(seeder.indexGardenStreet(O), full, 'the same after a rebuild') + + const owner = seeder._served.get(O) + owner.indexPaused = true + t.alike(seeder.indexGardenStreet(O), { items: [], next: null, indexing: false, complete: false }, 'a paused owner: no items') + owner.indexPaused = false + + seeder._served.get(A).searchHold = 1 + t.alike(seeder.indexGardenStreet(O), { ...full, items: [{ key: B, on: 1, state: null }], complete: false }, 'a held balcony is left out and the page is not complete') + seeder._served.get(A).searchHold = 0 + seeder._isBlocked = (hex) => hex === A + t.alike(seeder.indexGardenStreet(O), { ...full, items: [{ key: B, on: 1, state: null }], complete: true }, 'a blocked balcony is filtered, complete unchanged') + seeder._isBlocked = (hex) => hex === O + t.is(seeder.indexGardenStreet(O), null, 'a blocked owner is unknown') + seeder._isBlocked = () => false + owner.searchHold = 1 + t.is(seeder.indexGardenStreet(O), null, 'a held owner is unknown') + owner.searchHold = 0 + + await grow(O, ofO, [followRecord({ key: A, on: false, at: at(4) })]) // O:4 + t.alike(seeder.indexGardenStreet(O).items, [{ key: B, on: 1, state: null }], 'an unfollow takes A out') + seeder._drop(B, 'blocked') + t.alike(seeder.indexGardenStreet(O).items, [], 'a purged balcony leaves at once') + seeder._drop(O, 'blocked') + t.is(seeder.indexGardenStreet(O), null, 'a purged owner is unknown') +}) + +test('indexGardenStreet: a followed key the follows index hides is never an item or next where the key budget ends', async (t) => { + const { gardenRecord, followRecord } = require('@varanda/core/lib/records') + const { GARDEN_STREET_SCAN } = require('@varanda/core/lib/engagement-index') + const O = authorKey(1) + const ago = (h) => new Date(Date.now() - h * 3600000).toISOString() + const followed = [] + for (let i = 0; i < GARDEN_STREET_SCAN + 5; i++) followed.push(authorKey(100 + i)) + // H sits where the budget ended when every examined key counted. Its + // balcony is on, but O's follow of it is not shown once H is blocked. + const H = followed[GARDEN_STREET_SCAN - 1] + const ofO = followed.map((key, i) => followRecord({ key, on: true, at: at(i) })) + const { seeder, pass } = gardenFixture([[O, ofO], ...followed.map((key) => [key, key === H ? [gardenRecord({ on: true, at: ago(1) })] : []])]) + await pass() + t.alike(seeder.indexGardenStreet(O).items, [{ key: H, on: 1, state: null }], 'shown while H is not blocked') + seeder._isBlocked = (hex) => hex === H + const page = seeder.indexGardenStreet(O) + t.alike(page.items, [], 'H is not an item') + t.not(page.next, H, 'H is not next') + t.is(page.next, followed[GARDEN_STREET_SCAN], 'next is the last shown key the budget reached') + t.is(page.complete, true) + t.alike(seeder.indexGardenStreet(O, page.next).items, []) + t.is(seeder.indexGardenStreet(O, page.next).next, null, 'the walk ends') +}) + +test('indexGarden and indexGardenStreet: complete false while the follow index is incomplete for the owner', async (t) => { + const { gardenRecord, gardenHelpRecord, followRecord } = require('@varanda/core/lib/records') + const A = authorKey(1) + const B = authorKey(2) + const ago = (h) => new Date(Date.now() - h * 3600000).toISOString() + const ofA = [gardenRecord({ on: true, at: ago(5) }), followRecord({ key: B, on: true, at: at(0) })] + const ofB = [followRecord({ key: A, on: true, at: at(1) }), gardenHelpRecord({ to: A, pot: 0, at: ago(1) }), gardenRecord({ on: true, at: ago(2) })] + const { seeder, pass } = gardenFixture([[A, ofA], [B, ofB]]) + await pass() + t.alike(seeder.indexGarden(A), { on: 1, state: null, helps: [{ author: B, seq: 2, followSeq: 1 }], given: [], indexing: false, complete: true }) + t.alike(seeder.indexGardenStreet(A), { items: [{ key: B, on: 3, state: null }], next: null, indexing: false, complete: true }) + const real = seeder._notices.followersIncomplete + seeder._notices.followersIncomplete = () => { throw new Error('x') } + t.is(seeder.indexGarden(A).complete, false, 'a throwing followersIncomplete reads incomplete') + seeder._notices.followersIncomplete = real + const realFollowing = seeder._notices.followingIncomplete + seeder._notices.followingIncomplete = () => { throw new Error('x') } + t.is(seeder.indexGardenStreet(A).complete, false, 'a throwing followingIncomplete reads incomplete') + seeder._notices.followingIncomplete = realFollowing + seeder._notices.markPending(B) + t.alike(seeder.indexGarden(A), { on: 1, state: null, helps: [], given: [], indexing: false, complete: false }, 'a pending helper follow omits its help and says so') + seeder._notices.markPending(A) + t.alike(seeder.indexGardenStreet(A), { items: [], next: null, indexing: false, complete: false }, 'pending own follows: a short street says so') +}) + +test('indexProfile: garden pointer is the newest live garden block, on or off, deletes fall back, fail closed', async (t) => { + const { gardenRecord } = require('@varanda/core/lib/records') + const O = authorKey(1) + const ago = (h) => new Date(Date.now() - h * 3600000).toISOString() + const ofO = [gardenRecord({ on: true, at: ago(2) })] + const { seeder, pass, grow } = gardenFixture([[O, ofO]]) + await pass() + t.is(seeder.indexProfile(O).garden, 1, 'a garden block gives its seq') + await grow(O, ofO, [gardenRecord({ on: false, at: ago(1) })]) // 2 + t.is(seeder.indexProfile(O).garden, 2, 'an off block is still the pointer') + await grow(O, ofO, [deleteRecord({ seq: 2, at: at(0) })]) // 3 + t.is(seeder.indexProfile(O).garden, 1, 'a deleted block falls back') + seeder._rebuildIndex() + t.is(seeder.indexProfile(O).garden, null, 'between a rebuild and the re-index: fail closed') + await settleIndex(seeder) + t.is(seeder.indexProfile(O).garden, 1, 'the same after a rebuild') +}) + test('indexScraps: the caps count admitted writers only and pages carry exclusive cursors', async (t) => { const { scrapRecord, followRecord } = require('@varanda/core/lib/records') const O = authorKey(1) diff --git a/docs/changelog/feat-varandinha-index.md b/docs/changelog/feat-varandinha-index.md new file mode 100644 index 00000000..dfced970 --- /dev/null +++ b/docs/changelog/feat-varandinha-index.md @@ -0,0 +1,3 @@ +### Adicionado + +- Índice da Varandinha no semeador (PR `feat/varandinha-index`): o semeador passa a indexar os registros `garden`, `garden-state` e `garden-help`, guardando só chaves, seqs, a data de cada rega e o número do vaso regado, nunca o estado da varanda nem a planta, e responde duas rotas novas, `GET /index/garden/` (a varanda: os últimos `garden` e `garden-state`, as regas de vizinhos que seguem o dono dentro da janela de 72 h, uma por vizinho por dia de Brasília, no máximo 24, até 4 por vaso, e as regas que o próprio dono deu) e `GET /index/garden//rua[/]` (Varandas da rua: as varandas abertas de quem o dono segue, 20 por página, em ordem de chave). `/index/profile/` passa a levar `garden` (seq ou `null`). As duas rotas usam o mesmo orçamento das outras rotas `/index` (`INDEX_RATE`, sem limitador novo); um caminho malformado depois do dono é 400 e uma resposta fora do formato é 404. Ordem de lançamento: semeadores antes da web hospedada e dos apps. Conferência em `docs/verification/feat-varandinha-index.md`. diff --git a/docs/protocol.md b/docs/protocol.md index 7740ec0f..c95decb2 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -1191,6 +1191,19 @@ A member's device cannot see who wrote recados on a page without reading every d - **Privacy.** The seeder learns which pages' recados are opened, also when a node checks who may write before it writes a recado (the path carries the owner key, like the other `/index` routes) and, with `GET /index/follow//`, which follow pairs a node checked. It learns no text. - **Not hosted.** With manual configuration (`varanda-serve`) there is no seeder index, so no recados beyond what the node reads itself. +### Seeder Varandinha index (Varandinha; `packages/core/lib/engagement-index.js`, `packages/core/lib/seeder.js`) + +A member's device cannot find the newest `garden` and `garden-state` of a balcony, nor the waterings other members wrote to it, without reading every diary, so the seeder points at the records (section 2, "Varandinha"). It holds no pot, plant, act or name, only keys, seqs and each help's addressee, date and pot index, and the node reads and verifies every block itself (section 7 "Seeder index route", `GET /index/garden/` and `GET /index/garden//rua`). + +- **Inputs.** `INDEX_TYPES` includes `garden`, `garden-state` and `garden-help`; they count in the same record budget and per-author cap, in memory only, rebuilt at start. Every live block of these types holds a record slot under its writer and frees it when deleted, so latest-wins flags and states never pile up. Per author the index keeps the seqs of the live `garden` blocks (with their `on`) and `garden-state` blocks; the newest of each is the pointer, and a deletion falls back to the previous live block. A `garden-help` is kept under its writer and under its addressee (`to`) with its parsed date and its pot index; a help addressed to its own writer takes no slot and is never listed. Help rows belong to the writers' diaries, so they are kept across a drop of the addressee's diary, like `incoming`. +- **Lists.** `/index/garden/` answers `{ on, state, helps, given, indexing, complete }`. `on` is the seq of the owner's newest live `garden` block, on or off (the node peeks it and checks `on === true`), `state` its newest live `garden-state` seq, each `null` when there is none. When that newest `garden` block is off, or there is none, the answer stops there: `state` is `null` and `helps` and `given` are empty, so a member who switched the Varandinha off, or never opened it, is not one lookup away from the list of who watered them. `helps` are `[{ author, seq, followSeq }]`: only from writers whose `followSeq`, the seq of the HELPER's committed live `follow` of the owner, is a seq (a help from someone who does not follow the owner counts for nothing, so it is left out), only rows dated inside the reader window of now (not before `now - GARDEN_HELP_WINDOW_MS`, not after `now + GARDEN_MAX_FUTURE_MS`, and a Brasília day start that is a valid time), and one row per writer per Brasília day, the lowest seq of that day. The rows are then picked per pot, by the pot of each writer's row for that day: at most `GARDEN_HELP_ROWS / GARDEN_POTS` (4) rows per pot, every writer's first row in that pot taken first and only then that pot's other rows, so the answer holds at most `GARDEN_HELP_ROWS` (24) rows. A help counts only for a pot planted on or before it, so helps naming one pot, or an empty one, cannot push out the helps of another. Inside a pot, and in the answer, the order is: rows dated at or before the seeder's clock newest date first, then author ascending, then seq ascending; rows dated after it (up to `GARDEN_MAX_FUTURE_MS` ahead) after all of those, nearest to now first, then author, then seq, so writers who date their helps ahead cannot push honest helpers out. `given` are the owner's own live `garden-help` seqs to other balconies inside the same window, newest first, at most 24. `/index/garden//rua[/]` (Varandas da rua) answers `{ items: [{ key, on, state }], next, indexing, complete }`: it walks the owner's committed live follow targets in key order, strictly after the cursor key, skips every key the follows page would omit (blocked, not admitted, not gated or held) without counting it, and lists the keys whose newest live `garden` block is on (`on` that block's seq, `state` the newest live `garden-state` seq or `null`), at most `GARDEN_STREET_PAGE` (20) rows per page. `next` is the last key examined that the follows page would show, while keys remain, else `null`, so a page of closed balconies never dead-ends a walk; when only omitted keys remain after it, the next page is empty with `next: null`. +- **Caps.** Help rows are retained per (writer, owner) at most 4 (`GARDEN_PAIR_MAX`) and per writer at most 64 (`GARDEN_WRITER_MAX`), in the writer's append (seq) order, never by the date the record claims, so a far-future date cannot hold a slot that is never listed. Past either cap one row leaves and frees its slot: first a row that can never be listed (a later seq of a Brasília day that already has an earlier one, or a date outside the Date range), the highest seq of those, otherwise the lowest seq; when that row is the incoming one it is skipped without a slot. The scan passes its clock, and with it a row that cannot be listed at that instant (outside the reader window, or a day start that is not a valid time) is the first to leave a pair or writer cap, and the lowest seq of a Brasília day is taken among the listable rows only, as the reader does. At most 512 writers per owner (`GARDEN_WRITERS_PER_OWNER`), and a place there is held only by rows that can still be listed: a new writer past the cap whose row cannot be listed at the scan's clock is skipped without a slot; otherwise it takes the place of a writer none of whose rows to that owner can be listed (the one whose newest row is the oldest, a tie by the lower key), whose rows all leave and free their slots. When no place can be taken the new row is skipped, and since a skipped row is never replayed the owner is marked full until that row would leave the reader window (its date plus `GARDEN_HELP_WINDOW_MS`); a restart clears the mark with the rest of the index. A help that evicts a row of its own writer is net zero and passes a full record budget and per-author cap. One street call examines at most 2000 keys the follows page would show (`GARDEN_STREET_SCAN`; omitted keys do not count) and then answers a `next`. +- **Honesty flags.** As the other indexes: `indexing` and `complete`. The garden answer says `complete: false` when a writer with a listable row is one `visible` refuses, while the owner is marked full, and while the follow index may miss a follow of the owner (a helper whose follow is pending or was cut at a cap has no `followSeq`, so its help is left out). The street answer says it for a key held for a pending re-gate and while the owner's own follows are pending, truncated or capped (the street may be short). A blocked, not admitted or not gated key is dropped with `complete` unchanged, the omit rule of the follows page. +- **Omit rule.** A `null` from the seeder is the generic 404, never an empty balcony: an owner that is not served, gated, admitted and unblocked, one held for a pending re-gate and a purged one. A malformed street cursor, or any other bad tail under `/index/garden//`, is `400` `'pedido inválido'` and never reaches the seeder (section 7). While the owner's own diary is still being indexed or scanned (its newest blocks, tombstones or unfollows may be missing) the garden answer is `{ on: null, state: null, helps: [], given: [], indexing: true, complete: false }` and the street answer has no items, `next: null` and `complete: false`, so nothing stale is listed. An off or never opened balcony answers no helps, no given and no state (`on` still names the newest `garden` block or is `null`). A client treats `complete: false` as unknown. +- **Trust model.** Every pointer is a hint. The node re-reads each pointed block with `diary.peek`, checks it with `validateRecord` and folds it with the rules of section 2 "Varandinha" (`foldHelps`, `foldGiven`, `foldBalcony`), including each helper's follow block (`isFollowOf`): a help counts only when the writer's own diary holds a live `garden-help` addressed to the owner and the helper's own live `follow` of the owner. A lying seeder can hide pointers, name an older authentic `garden` or `garden-state` block, or name a follow the helper has since undone (one peek proves a block is live, not that it is the latest), so it can make a balcony look older, closed or less helped. It cannot invent a help: a pointer to a block that is not a live `garden-help` to that owner, or not backed by the helper's follow, fails verification and counts nothing, and the fold caps what any list can add. +- **Privacy.** The seeder learns which balcony is opened (the path carries the owner key, like the other `/index` routes) and whose street is walked. The street lists only keys the owner follows that show their balcony (a public `garden` block that is on); its `next` cursor is always a key the owner follows that `GET /index/follows//seguindo` would show (it may have a closed balcony), never a follow that index omits. It learns no pot or plant beyond the public blocks every reader can fetch. +- **Not hosted.** With manual configuration (`varanda-serve`) there is no seeder index, so no help pointers and no street beyond what the node reads itself from diaries it already follows. + ### Seeder profile index (Perfil rico; `packages/core/lib/engagement-index.js`, `packages/core/lib/seeder.js`) A member's device cannot find the newest `profile-about` and `mood` of an author without reading the whole diary, so the seeder points at them (section 2, "Perfil rico"). It holds only seqs, and the node reads and verifies each block itself (section 7 "Seeder index route", `GET /index/profile/`). @@ -1202,6 +1215,7 @@ A member's device cannot find the newest `profile-about` and `mood` of an author - **Trust model.** Which seq is the newest, and whether one exists, are claims of the seeder. A member node shows only what it re-read with `diary.peek` and checked with the rules of section 2: a seeder can point at an older authentic `profile-about` or `mood` or omit one, never make a forged or invalid one show, and never a `mood` older than 24 hours. - **Privacy.** The seeder learns which profiles are opened (the path carries the author key, like the other `/index` routes). It learns no text. - **Visitas pointer.** The answer also carries `visits: seq | null`, the highest live `visits` block of the author (section 11, "Visitas"). It is a pointer to a block that may say off; the node peeks it. +- **Varandinha pointer.** The answer also carries `garden: seq | null`, the highest live `garden` block of the author, on or off (section 2, "Varandinha"; section 5 "Seeder Varandinha index"). It is a pointer only: an off block still has a seq, so the node peeks the block, verifies it and checks `on === true`. While the author's own diary is pending it is `null`, like the other pointers. - **Not hosted.** With manual configuration (`varanda-serve`) there is no seeder index, so no perfil rico beyond what the node reads itself. ### Seeder fan index (Fãs e Melhores amigos; `packages/core/lib/engagement-index.js`, `packages/core/lib/seeder.js`) @@ -1691,7 +1705,9 @@ The same listener answers the engagement index of section 5 ("Seeder engagement - `/index/follows//(seguindo|seguidores)[/.]`, regex `^\/index\/follows\/([0-9a-f]{64})\/(seguindo|seguidores)(?:\/([0-9]{1,16}\.[0-9a-f]{64}))?$` (`INDEX_FOLLOWS_RE`). The cursor is `.` (no seq: a follow pair is unique by key). -> `Seeder#indexFollows`. Answer `{ author, kind, items: [{ key, seq, at }], next: { atMs, key } | null, total, authors, indexing, complete }`: at most 20 items, ordered by the follow time clamped at 0 (descending), then key; `seguidores` rows point at the follower's follow block and `seguindo` rows at the author's own; `total` counts visible rows only. Same 404 policy as the other `/index` routes. Section 5 "Seeder follow index". - `/index/testimonials/`, regex `^\/index\/testimonials\/([0-9a-f]{64})$` (`INDEX_TESTIMONIALS_RE`). -> `Seeder#indexTestimonials`. Answer `{ shown: [{ seq, from, wseq }], pending: [{ from, seq }], indexing, complete }`: `shown` rows point at the owner's `testimonial-show` block (`seq`) and the writer's `testimonial` block (`from`, `wseq`), `pending` rows at a writer's `testimonial` block; at most 64 of each, newest first. Pointers only, never the text or the sealed box: the client reads every block with `diary.peek` and verifies it (section 2, "Depoimentos"; docs/rpc.md 5.100). Same Host check, 405, generic 404 policy and caps as the other `/index` routes. Section 5 "Seeder testimonial index". - `/index/scraps/[/]`, regex `^\/index\/scraps\/([0-9a-f]{64})(?:\/([0-9]{1,16}\.[0-9a-f]{64}\.[1-9][0-9]{0,14}))?$` (`INDEX_SCRAPS_RE`). -> `Seeder#indexScraps`. Answer `{ scraps: [{ author, seq, followSeq }], hides: [seq], wall: seq | null, next: string | null, indexing, complete }`: at most 50 rows per page (`author` hex64, `seq` and `followSeq` integers >= 1, `followSeq` `null` when admitted by `'everyone'` or the reply exception), already filtered by the wall, the hides, the reply exception and the caps (20 per writer, 500 per page, over admitted rows only), order `at` descending, author ascending, seq descending; `hides` at most 1000 seqs of the owner's live `scrap-hide` blocks, `wall` the seq of the owner's latest live `scrap-wall` or `null`. The cursor is `..` and `next` has the same shape or is `null`. Any other segment after the owner is `400` `'pedido inválido'`, never `404`, so a client tells a bad cursor from an unknown owner; an unknown owner is `404`. Pointers only, never the text: the client reads each block with `diary.peek` and verifies it, including each follow pointer. Same Host check, 405, 404 policy and `INDEX_RATE` budget as the other `/index` routes. Section 5 "Seeder recados index". - - `/index/profile/`, regex `^\/index\/profile\/([0-9a-f]{64})$` (`INDEX_PROFILE_RE`). -> `Seeder#indexProfile`. Answer `{ about: integer >= 1 | null, mood: integer >= 1 | null, fans: { count: integer >= 0, indexing } | null, top: { seq: integer >= 1, back: { : integer >= 1 | null } } | null, indexing, complete }`: the seq of the author's newest live `profile-about` and `mood` block, the claimed number of fans, and the newest live `top-friends` block with, per listed friend (at most 8), the seq of that friend's live `follow` of the author or `null` (section 5 "Seeder fan index"). The object is open (section 5 "Seeder profile index"): later features add pointers, and a reader ignores keys it does not know. While the author's diary is not fully indexed, `about`, `mood` and `top` are `null`, `fans` keeps the known count with `indexing: true`, `indexing` is true and `complete` false. Pointers only, never a text; the node verifies each block. A 404 `não encontrado` for an author that is not served, gated, admitted and unblocked, one held for a pending re-gate, a purged one, a malformed key and a `null` answer. It uses the same index budget as the other routes (in flight, rate bucket and the Caddy limit below). + - `/index/garden/`, regex `^\/index\/garden\/([0-9a-f]{64})$` (`INDEX_GARDEN_RE`). -> `Seeder#indexGarden`. Answer exactly `{ on: integer >= 1 | null, state: integer >= 1 | null, helps: [{ author, seq, followSeq }], given: [integer >= 1], indexing: boolean, complete: boolean }`: `on` the seq of the owner's newest live `garden` block, on or off (an off block still has a seq; the node peeks it and checks `on === true`), `state` the newest live `garden-state` seq (`null`, with no helps and no given, while that `garden` block is off or there is none); at most 24 (`GARDEN_HELP_ROWS`) help rows of exactly `{ author, seq, followSeq }` (`author` hex64 and never the owner, `seq` and `followSeq` integers >= 1, `followSeq` the helper's live `follow` of the owner), no `author:seq` twice; at most 24 distinct `given` seqs (the owner's own helps). Any other answer (an extra or missing key, a duplicate, an oversized list, a non-boolean flag) is the generic 404, never a trimmed 200. Pointers only, never a pot or a plant: the client reads each block with `diary.peek` and verifies it, including each follow pointer. Section 5 "Seeder Varandinha index". + - `/index/garden//rua[/]`, regex `^\/index\/garden\/([0-9a-f]{64})\/rua(?:\/([0-9a-f]{64}))?$` (`INDEX_GARDEN_STREET_RE`). -> `Seeder#indexGardenStreet`. Answer exactly `{ items: [{ key, on, state }], next: hex64 | null, indexing: boolean, complete: boolean }`: at most 20 (`GARDEN_STREET_PAGE`) rows of exactly `{ key, on, state }` (`key` hex64, distinct, `on` integer >= 1, the seq of that key's newest live `garden` block, which is on, `state` integer >= 1 or `null`), keys ascending strictly after the cursor key; `next` is the cursor of the following page or `null`. Any other answer is the generic 404. Any other segment after `/index/garden//` (a malformed cursor, a trailing slash, another word) is `400` `'pedido inválido'`, never `404`, so a client tells a bad cursor from an unknown owner; an unknown owner is `404`. Both garden routes have the same Host check, 405, 404 policy and `INDEX_RATE` budget as the other `/index` routes, with no limiter of their own. + - `/index/profile/`, regex `^\/index\/profile\/([0-9a-f]{64})$` (`INDEX_PROFILE_RE`). -> `Seeder#indexProfile`. Answer `{ about: integer >= 1 | null, mood: integer >= 1 | null, fans: { count: integer >= 0, indexing } | null, top: { seq: integer >= 1, back: { : integer >= 1 | null } } | null, visits: integer >= 1 | null, garden: integer >= 1 | null, indexing, complete }`: the seq of the author's newest live `profile-about` and `mood` block, the newest live `visits` and `garden` blocks (pointers only: an off block still has a seq, so the node peeks it and checks `on === true`; section 11 "Visitas" and section 5 "Seeder Varandinha index"), the claimed number of fans, and the newest live `top-friends` block with, per listed friend (at most 8), the seq of that friend's live `follow` of the author or `null` (section 5 "Seeder fan index"). The object is open (section 5 "Seeder profile index"): later features add pointers, and a reader ignores keys it does not know. While the author's diary is not fully indexed, `about`, `mood`, `top`, `visits` and `garden` are `null`, `fans` keeps the known count with `indexing: true`, `indexing` is true and `complete` false. Pointers only, never a text; the node verifies each block. A 404 `não encontrado` for an author that is not served, gated, admitted and unblocked, one held for a pending re-gate, a purged one, a malformed key and a `null` answer. It uses the same index budget as the other routes (in flight, rate bucket and the Caddy limit below). - `/index/fans/[/.]`, regex `^\/index\/fans\/([0-9a-f]{64})(?:\/([0-9]{1,16}\.[0-9a-f]{64}))?$` (`INDEX_FANS_RE`). -> `Seeder#indexFans`. Answer `{ author, items: [{ key, seq, at }], next: { atMs, key } | null, total, authors, indexing, complete }`, the fields of the `seguidores` list without `kind`: at most 20 items, newest first by the fan's own `at` clamped at 0, then key, the cursor `.` exclusive, `seq` and `at` pointing at the fan's own `fan` block (section 5 "Seeder fan index"). A malformed row, a duplicate key or the author itself is a 404, as are an author that is not served, gated, admitted and unblocked, one held for a pending re-gate and a purged one. The node verifies each row with `diary.peek` of the fan's diary. It uses the same index budget as the other routes. - `/index/follow//`, regex `^\/index\/follow\/([0-9a-f]{64})\/([0-9a-f]{64})$` (`INDEX_FOLLOW_RE`). -> `Seeder#indexFollowSeq`. Answer `{ seq: integer >= 1 | null, indexing }`: the seq of `author`'s live `follow` record of `target`. A hint like the rest; the node verifies the block. - `/index/poll//`, regex `^\/index\/poll\/([0-9a-f]{64})\/([1-9][0-9]{0,14})$` (`INDEX_POLL_RE`). -> `Seeder#indexPoll`. Answer `{ totals: [integer >= 0, ...], indexing, complete }`, 2 to 4 totals (the claimed votes per option, section 5 "Seeder poll index"), or `{ totals: [], indexing: true, complete: false }` while the author's own diary is still being indexed. Rebuilt field by field, a non-array, more than 4 totals or a bad count is a 404. A 404 `não encontrado` for an author that is not served, gated, admitted and unblocked, one held for a pending re-gate, a post that is not a live poll post (a deleted one included) and a malformed seq. It uses the same index budget as the other routes. diff --git a/docs/verification/feat-varandinha-index.md b/docs/verification/feat-varandinha-index.md new file mode 100644 index 00000000..92241db3 --- /dev/null +++ b/docs/verification/feat-varandinha-index.md @@ -0,0 +1,59 @@ +# Varandinha, índice do semeador (PR feat/varandinha-index) + +Este PR acrescenta ao semeador o índice da Varandinha: as leituras `garden()` e `gardenStreet()` e a chave `garden` de `profileExtra()` no índice de engajamento (`packages/core/lib/engagement-index.js`), os tipos `garden`, `garden-state` e `garden-help` em `INDEX_TYPES`, os métodos `Seeder#indexGarden` e `Seeder#indexGardenStreet` e a chave `garden` de `Seeder#indexProfile` (`packages/core/lib/seeder.js`), as rotas `GET|HEAD /index/garden/` e `/index/garden//rua[/]` com os validadores `gardenBody` e `gardenStreetBody` e a chave `garden` em `profileBody` (`apps/cli/lib/seed-ws.js`), a subseção "Seeder Varandinha index" na seção 5 de `docs/protocol.md`, os dois itens de rota na seção 7 e a chave `garden` no texto do índice de perfil e no item da rota de perfil. + +## Rotas e validadores (`apps/cli/lib/seed-ws.js`) + +Rodado no Node 24.22.1, sobre a14de943 (a ponta de feat/varandinha-records, o PR de baixo da pilha), com as mudanças deste PR ainda fora de commit: + +- `npx standard apps/cli/lib/seed-ws.js apps/cli/test/seed-ws.test.js` + resultado: passou. +- `npm test -w apps/cli -- test/seed-ws.test.js` (e `node apps/cli/test/seed-ws.test.js`, o mesmo arquivo sozinho) + resultado: passou, 106/106 testes e 2430/2430 asserções, 1 arquivo, 0 falha. + +O que os testes novos de `apps/cli/test/seed-ws.test.js` cobrem: + +- `/index/garden/` e `/index/garden//rua[/]`: o texto exato das duas regex exportadas, a resposta 200 com cabeçalhos JSON, HEAD com o mesmo `Content-Length` e corpo vazio, o cursor que chega ao semeador como veio, dono desconhecido 404 nas duas rotas, caminhos malformados 404 (`/index/garden`, chave curta, longa ou maiúscula, `?x=1`), cauda inválida depois do dono 400 `pedido inválido` com `no-store` (`/`, `/x`, `/rua/`, `/Rua`, `/ruas`, cursor maiúsculo, curto, longo, com barra no fim, com segmento a mais, cursor de recados, uma chave sem `rua`), nenhum caminho malformado chega ao semeador, Host errado 421, e POST, PUT, DELETE e PATCH 405 com `Allow: GET, HEAD`. +- Resposta fora do formato exato é 404 `não encontrado`, nunca um 200 aparado: chave a mais ou a menos, chave a mais numa linha, ajuda sem `followSeq` ou com `followSeq` nulo, ajuda do próprio dono, linha repetida, seq 0 ou em texto, autor maiúsculo, linha em array, 25 ajudas, 25 `given`, `given` repetido, 0 ou fracionário, `on` 0 ou booleano, `state` em texto ou fora do inteiro seguro, bandeiras não booleanas; na rua, chave `rows` no lugar de `items`, chave a mais no item, item sem `state`, `on` nulo, `state` 0, chave repetida ou maiúscula, 21 itens, `items` nulo, `next` em formato de cursor de recados ou numérico. Um throw do semeador é 404 sem eco. +- O pior caso de cada rota (24 ajudas e 24 `given` com seqs no máximo inteiro seguro; 20 itens) cabe em `INDEX_MAX_BODY`, e a resposta de dono pendente passa. As duas rotas gastam o mesmo balde `/index` (a terceira chamada a outra rota recebe 503 com `Retry-After: 1`). +- `/index/profile/`: as asserções de formato exato agora incluem `garden` (padrão `null`, pior caso no máximo inteiro seguro), e `garden` 0, fracionário, em texto, negativo ou booleano é 404. + +Conferido que os testes falham sem a correção: com a checagem de chaves exatas da linha de ajuda e a recusa de ajuda do próprio dono tiradas de `gardenBody`, o arquivo falhou em "garden: an extra help key -> 404" e "garden: a help from the owner -> 404" (105/106 testes, 2426/2430 asserções); com as checagens de volta, passou de novo. + +Conferido à mão que as rotas, as regex e as chaves JSON descritas na seção 7 de `docs/protocol.md` são as do código, e que não há travessão (U+2014) nas linhas novas. + +## Índice e métodos do semeador (`engagement-index.js`, `seeder.js`, `notices.js`) + +Rodado no Node 24.22.1, na mesma base, depois das correções da revisão e da auditoria: + +- `npx standard packages/core/lib/engagement-index.js packages/core/lib/seeder.js packages/core/lib/notices.js packages/core/test/garden-index.test.js apps/cli/test/seeder-index.test.js` + resultado: passou. +- `npm test -w packages/core -- test/garden-index.test.js test/profile-extra-index.test.js test/engagement.test.js test/scraps-index.test.js test/notices.test.js` + resultado: passou, 5 arquivos, 0 falha; `garden-index.test.js` 36/36 testes e 6639/6639 asserções, `profile-extra-index.test.js` 14/14, `engagement.test.js` 24/24, `scraps-index.test.js` 13/13, `notices.test.js` 31/31. +- `npm test -w apps/cli -- test/seeder-index.test.js test/seed-ws.test.js` + resultado: passou, 2 arquivos, 0 falha; `seeder-index.test.js` 53/53 testes e 9052/9052 asserções, `seed-ws.test.js` 106/106. + +O que os testes de `packages/core/test/garden-index.test.js` e `apps/cli/test/seeder-index.test.js` cobrem, além do que já estava acima: + +- Varanda desligada ou nunca aberta: nenhuma rega, nenhum `given` e `state` nulo, sem chamar `visible` nem `followSeq`. +- Tetos por par (4), por escritor (64) e por dono (512 escritores): com o relógio do semeador, um escritor cujas linhas já não podem ser listadas dá lugar ao novo; uma linha que nunca será listada não ocupa lugar; uma rega recusada que ainda poderia ser listada marca a varanda como cheia, e a resposta diz `complete: false` até a janela dela passar. Uma rega que troca outra do mesmo escritor não é recusada pelo orçamento. +- Ordem: regas datadas depois do relógio do semeador ficam por último, e no máximo 4 regas por vaso, com cada escritor antes do segundo dia de outro, de modo que regas num vaso vazio ou num só vaso não tiram da lista as regas honestas de outro. +- Seguidores pendentes, cortados ou no teto deixam a resposta da varanda com `complete: false`; seguidos pendentes deixam a rua com `complete: false`. +- Rua: uma chave seguida que o índice de seguidos esconde nunca aparece nos itens nem no cursor `next`, também onde o orçamento de 2000 chaves acaba. + +Conferido que cada teste de regressão falha sem a correção (revertendo a correção numa cópia temporária e rodando de novo), exceto dois que fixam um comportamento que já valia: a escolha de uma linha por escritor e dia entre vasos, e o fim da rua quando só restam chaves escondidas. + +## CI completa + +Rodado no Node 24.22.1, sobre 30d2f89e (feat/varandinha-records com a `main` f25dc48b mesclada), com as mudanças deste PR por cima: + +- `npm ci --ignore-scripts` + resultado: passou. +- `npm run lint` + resultado: passou. +- `npm test -- --affected=origin/main` + resultado: 812 arquivos de teste, 811 passaram e 1 falhou por tempo esgotado: `apps/cli/test/update.test.js` parou no teste "update.sh: up to date with an intervention, a floor that cannot be written does not blame the health gate", com os 64 testes anteriores passando. Rodado sozinho (`node apps/cli/test/update.test.js`), passou, 73/73 testes e 1901/1901 asserções. O arquivo não toca nada deste PR; a mesma falha por tempo apareceu na CI do PR de baixo e também passou sozinha. +- `npm run demo` + resultado: DEMO OK. + +Não rodado: o Node 20 da CI (só o Node 24 local), aparelhos e navegador (este PR não tem tela). diff --git a/packages/core/lib/engagement-index.js b/packages/core/lib/engagement-index.js index 867c01bd..f43b5976 100644 --- a/packages/core/lib/engagement-index.js +++ b/packages/core/lib/engagement-index.js @@ -103,8 +103,29 @@ // writer, then SCRAP_PER_PAGE per page, newest first (at desc, author asc, // seq desc). Each request examines at most SCRAP_SCAN pointers (writers // included); past it the answer says truncated. +// +// Varandinha (docs/protocol.md, the rules are @varanda/rpc lib/garden.js): +// every live `garden`, `garden-state` and `garden-help` block holds a record +// slot under its writer and is freed when deleted, so latest-wins flags and +// states never pile up. Only seqs and the help's addressee, date and pot +// index are kept, never the state's pots, plants or acts. The newest live `garden` block (on or +// off) and `garden-state` block are pointers (garden(), profileExtra()); a +// deletion falls back to the previous one. Helps are kept per (writer, +// addressee): GARDEN_PAIR_MAX per pair, GARDEN_WRITER_MAX per writer (past +// either, one row leaves and frees its slot, or the incoming row is skipped +// without a slot when it is the one helpVictim names) and +// GARDEN_WRITERS_PER_OWNER writers per addressee. A place at that last cap +// is held only by rows garden() can still list: with the scan's clock +// (add opts.now), a new writer past it takes the place of a writer none of +// whose rows is listable, and its own row is skipped without a slot when +// it is not listable either. When no place can be taken (or there is no +// clock) the row is skipped, and garden() says hidden until it would have +// left the reader window, since a skipped row is never replayed. A help +// addressed to its own writer takes no slot and is never listed. Help rows +// are kept across a drop of the addressee's diary (they belong to the +// writers' diaries), like `incoming`. const { formatRef, parseRef, MAX_TOP_FRIENDS, POLL_MIN, POLL_MAX } = require('./records') -const { isEvent } = require('@varanda/rpc') +const { isEvent, garden: gardenRules } = require('@varanda/rpc') const { commitment } = require('./testimonials') const { foldPicks } = require('./polls') @@ -144,6 +165,14 @@ const EVENT_RSVP_OPTIONS = 2 // Distinct polls one voter may hold operations on, and voters one poll lists. const CHOICE_POLLS_MAX = 4096 const CHOICE_VOTERS_MAX = 20000 +// Varandinha help pointers: per (writer, addressee), per writer, writers per +// addressee, and the keys one gardenStreet() call examines. They bound the +// memory a flood of signed helps can take; the eviction rules (header) keep +// a full cap from shutting out a listable help for good. +const GARDEN_PAIR_MAX = 4 +const GARDEN_WRITER_MAX = 64 +const GARDEN_WRITERS_PER_OWNER = 512 +const GARDEN_STREET_SCAN = 2000 // Same value as lib/feed.js MAX_FUTURE_MS (not imported: this module only // requires ./records and ./testimonials). An engager edge dated further ahead is hidden, never // removed, and appears once the clock catches up. @@ -453,6 +482,82 @@ function validChoice (record) { (record.pick === null || (Number.isSafeInteger(record.pick) && record.pick >= 0 && record.pick < POLL_MAX)) && typeof record.at === 'string' } +// Stored garden-help rows: newest first (atMs desc), a tie by the lowest +// seq, the order in which the reader fold picks a helper's newest row. +function compareHelps (a, b) { + if (a.atMs !== b.atMs) return a.atMs > b.atMs ? -1 : 1 + return a.seq - b.seq +} + +// Whether garden(owner, now) may list a help dated atMs: inside the reader +// window (lib/garden.js inWindow) on a day dayStart accepts. One test for +// garden() and for the caps, so a cap never keeps a row garden() drops. +function gardenListable (atMs, now) { + return atMs <= now + gardenRules.GARDEN_MAX_FUTURE_MS && atMs >= now - gardenRules.GARDEN_HELP_WINDOW_MS && + !Number.isNaN(gardenRules.dayStart(atMs)) +} + +// The row a full help cap gives up among `rows` ([{ seq, atMs, to }], the +// kept rows and the incoming one). Retention follows the writer's own seq +// order, never the date it claims, so a far-future date cannot hold a slot +// that garden() never lists while every later help is refused. A row the +// reader fold would never count while the others stay goes first, the +// highest seq of those: not the lowest seq of its (addressee, Brasilia day), +// or a day dayStart refuses. Evicting the lowest seq of a day instead would +// let a later duplicate of that day be listed in its place. Otherwise the +// lowest seq leaves. With a clock (`now`, ms), a row garden(now) could not +// list is dead first and the lowest seq of a day is taken among the others +// only, as garden() and the reader fold filter the window before they pick +// the day's row: a day the window edge cuts keeps its listed row. +function helpVictim (rows, now = null) { + const days = rows.map((r) => gardenRules.dayStart(r.atMs)) + const alive = rows.map((r, i) => !Number.isNaN(days[i]) && (now === null || gardenListable(r.atMs, now))) + const lowest = new Map() + rows.forEach((r, i) => { + if (!alive[i]) return + const key = r.to + '/' + days[i] + const prev = lowest.get(key) + if (prev === undefined || r.seq < prev) lowest.set(key, r.seq) + }) + let dead = null + let oldest = null + rows.forEach((r, i) => { + if (!alive[i] || lowest.get(r.to + '/' + days[i]) !== r.seq) { + if (dead === null || r.seq > dead.seq) dead = r + } else if (oldest === null || r.seq < oldest.seq) { + oldest = r + } + }) + return dead ?? oldest +} + +// Rows by date, newest first, then author asc, then seq asc. garden() wraps +// it (gardenRowOrder) so a row dated after its clock never outranks one +// that is not. +function compareHelpRows (a, b) { + if (a.atMs !== b.atMs) return a.atMs > b.atMs ? -1 : 1 + if (a.author !== b.author) return a.author < b.author ? -1 : 1 + return a.seq - b.seq +} + +// garden(now)'s pick and answer order. A writer can date a help up to +// GARDEN_MAX_FUTURE_MS ahead and still be listed, so ranking by the claimed +// date alone would let a few writers dating ahead own every listed row. +// Rows dated after now therefore come after every row at or before now, +// nearest to now first, then author, then seq; the others keep +// compareHelpRows. +function gardenRowOrder (now) { + return (a, b) => { + const fa = a.atMs > now + const fb = b.atMs > now + if (fa !== fb) return fa ? 1 : -1 + if (!fa) return compareHelpRows(a, b) + if (a.atMs !== b.atMs) return a.atMs < b.atMs ? -1 : 1 + if (a.author !== b.author) return a.author < b.author ? -1 : 1 + return a.seq - b.seq + } +} + // Fan rows: atMs desc, then key asc (lib/notices.js compareRows). function compareFanRows (a, b) { if (a.atMs !== b.atMs) return a.atMs > b.atMs ? -1 : 1 @@ -516,6 +621,16 @@ function createEngagementIndex ({ maxRecords = DEFAULT_MAX_RECORDS, maxTargets = // on that post (ops is the voter's own list, ascending seq, shared with its // author entry), kept across a drop of the poll's own diary like fanIn. const pollIn = new Map() + // Addressee -> Map: the live + // `garden-help` blocks each writer addressed to that balcony (at most + // GARDEN_PAIR_MAX), the same rows as the writer's gardenHelps, kept across + // a drop of the addressee's diary like `incoming`. + const helpsByTo = new Map() + // Addressee -> fullUntil ms: the writers cap refused a help garden() might + // list, and a refused help is never replayed, so garden() says hidden + // until that help would have left the reader window. Kept across a drop + // of the addressee's diary like helpsByTo. + const gardenFull = new Map() function invalidateNotices (author) { noticeCache.delete(author) @@ -523,7 +638,7 @@ function createEngagementIndex ({ maxRecords = DEFAULT_MAX_RECORDS, maxTargets = function authorEntry (author) { let entry = authors.get(author) if (entry === undefined) { - entry = { count: 0, timeline: [], posts: new Map(), reposts: new Map(), touched: new Map(), profileSeqs: [], usernameSeqs: [], copiedSeqs: [], aboutSeqs: [], moodSeqs: [], engagements: new Map(), testimonials: null, shows: null, scraps: null, scrapsByTo: null, scrapHides: null, hideSeqs: null, hiddenRefs: null, walls: null, wallSeqs: null, fanSeqs: null, fanOps: null, tops: null, topSeqs: null, choiceSeqs: null, choiceOps: null, eventSeqs: null, visitSeqs: null } + entry = { count: 0, timeline: [], posts: new Map(), reposts: new Map(), touched: new Map(), profileSeqs: [], usernameSeqs: [], copiedSeqs: [], aboutSeqs: [], moodSeqs: [], engagements: new Map(), testimonials: null, shows: null, scraps: null, scrapsByTo: null, scrapHides: null, hideSeqs: null, hiddenRefs: null, walls: null, wallSeqs: null, fanSeqs: null, fanOps: null, tops: null, topSeqs: null, choiceSeqs: null, choiceOps: null, eventSeqs: null, visitSeqs: null, gardenSeqs: null, gardenOn: null, gardenStateSeqs: null, gardenHelps: null } authors.set(author, entry) } return entry @@ -633,7 +748,8 @@ function createEngagementIndex ({ maxRecords = DEFAULT_MAX_RECORDS, maxTargets = const type = record.type if (type !== 'post' && type !== 'like' && type !== 'repost' && type !== 'profile' && type !== 'username' && type !== 'testimonial' && type !== 'testimonial-show' && type !== 'scrap' && type !== 'scrap-hide' && - type !== 'scrap-wall' && type !== 'profile-about' && type !== 'mood' && type !== 'fan' && type !== 'top-friends' && type !== 'choice' && type !== 'visits') return false + type !== 'scrap-wall' && type !== 'profile-about' && type !== 'mood' && type !== 'fan' && type !== 'top-friends' && type !== 'choice' && type !== 'visits' && + type !== 'garden' && type !== 'garden-state' && type !== 'garden-help') return false if (type === 'testimonial' && (!HEX64.test(record.to) || !HEX64.test(record.hash))) return false if (type === 'scrap' && (!HEX64.test(record.to) || (record.replyTo !== null && (typeof record.replyTo !== 'object' || !HEX64.test(record.replyTo.author) || !isSeq(record.replyTo.seq))))) return false @@ -643,8 +759,14 @@ function createEngagementIndex ({ maxRecords = DEFAULT_MAX_RECORDS, maxTargets = if (type === 'top-friends' && !validTopKeys(record.keys)) return false if (type === 'choice' && !validChoice(record)) return false if (type === 'visits' && typeof record.on !== 'boolean') return false + if (type === 'garden' && typeof record.on !== 'boolean') return false + if (type === 'garden-state' && !Array.isArray(record.pots)) return false + const helpAtMs = type === 'garden-help' && typeof record.at === 'string' ? Date.parse(record.at) : NaN + if (type === 'garden-help' && (typeof record.to !== 'string' || !HEX64.test(record.to) || + !gardenRules.isPotIndex(record.pot) || !Number.isFinite(helpAtMs))) return false // Never self (lib/fans.js): as if never written, no slot. if (type === 'fan' && record.key === author) return true + if (type === 'garden-help' && record.to === author) return true if (type === 'top-friends' && record.keys.includes(author)) return true if (type === 'testimonial-show' && (!HEX64.test(record.from) || !isSeq(record.seq) || typeof record.text !== 'string' || typeof record.nonce !== 'string')) return false @@ -666,11 +788,56 @@ function createEngagementIndex ({ maxRecords = DEFAULT_MAX_RECORDS, maxTargets = indexOfSeq(existing.moodSeqs, seq) !== -1 || existing.testimonials?.has(seq) || existing.shows?.has(seq) || existing.scraps?.has(seq) || existing.scrapHides?.has(seq) || existing.walls?.has(seq) || existing.fanSeqs?.has(seq) || existing.tops?.has(seq) || existing.choiceSeqs?.has(seq) || - (existing.visitSeqs !== null && indexOfSeq(existing.visitSeqs, seq) !== -1))) return true + (existing.visitSeqs !== null && indexOfSeq(existing.visitSeqs, seq) !== -1) || + (existing.gardenSeqs !== null && indexOfSeq(existing.gardenSeqs, seq) !== -1) || + (existing.gardenStateSeqs !== null && indexOfSeq(existing.gardenStateSeqs, seq) !== -1) || + existing.gardenHelps?.has(seq))) return true if (type === 'fan' && existing?.fanOps?.has(record.key) !== true && (existing?.fanOps?.size ?? 0) >= FAN_KEYS_MAX) return true if (type === 'choice' && existing?.choiceOps?.has(pollRef) !== true && ((existing?.choiceOps?.size ?? 0) >= CHOICE_POLLS_MAX || (pollIn.get(pollRef)?.size ?? 0) >= CHOICE_VOTERS_MAX)) return true - if (records >= cap || (existing ? existing.count : 0) >= authorCap) return false + // The help cap skips (no slot) come before admission, like the fan + // cap; a kept help past a cap evicts the row helpEvict names (the same + // writer's) or every row of writer ownerEvict to this addressee. + // Everything is decided here and nothing changes until the checks + // below pass, so a refusal leaves no trace. + let helpEvict = null + let ownerEvict = null + if (type === 'garden-help') { + const now = typeof opts?.now === 'number' && Number.isFinite(opts.now) ? opts.now : null + const row = { seq, atMs: helpAtMs, to: record.to } + const writers = helpsByTo.get(record.to) + const list = writers?.get(author) + let victim = null + if (list !== undefined && list.length >= GARDEN_PAIR_MAX) { + victim = helpVictim([...list.map((r) => ({ seq: r.seq, atMs: r.atMs, to: record.to })), row], now) + } else if ((existing?.gardenHelps?.size ?? 0) >= GARDEN_WRITER_MAX) { + const kept = [...existing.gardenHelps].map(([s, info]) => ({ seq: s, atMs: info.atMs, to: info.to })) + victim = helpVictim([...kept, row], now) + } + if (victim !== null) { + if (victim.seq === seq) return true + helpEvict = victim.seq + } + if (list === undefined && (writers?.size ?? 0) >= GARDEN_WRITERS_PER_OWNER) { + // A row garden(now) cannot list costs no place. Otherwise the + // place of a writer none of whose rows garden(now) can list is + // taken, so stale or far-future rows never hold the cap forever. + if (now !== null && !gardenListable(helpAtMs, now)) return true + if (now !== null) ownerEvict = evictableWriter(writers, now) + if (ownerEvict === null) { + if (!Number.isNaN(gardenRules.dayStart(helpAtMs))) { + const until = helpAtMs + gardenRules.GARDEN_HELP_WINDOW_MS + 1 + if (!(gardenFull.get(record.to) >= until)) gardenFull.set(record.to, until) + } + return true + } + } + } + // An eviction of the same writer's row makes the add net zero for the + // global and the author's counts; another writer's frees global slots + // only. + if ((records >= cap && helpEvict === null && ownerEvict === null) || + ((existing ? existing.count : 0) >= authorCap && helpEvict === null)) return false // Off operations need no new target; they may free one at the cap. const needed = new Set([replyTo, quote, record.on === true ? to : null]) needed.delete(null) @@ -771,6 +938,23 @@ function createEngagementIndex ({ maxRecords = DEFAULT_MAX_RECORDS, maxTargets = } else if (type === 'visits') { if (entry.visitSeqs === null) entry.visitSeqs = [] entry.visitSeqs.splice(lowerBoundSeq(entry.visitSeqs, seq), 0, seq) + } else if (type === 'garden') { + if (entry.gardenSeqs === null) { + entry.gardenSeqs = [] + entry.gardenOn = new Map() + } + entry.gardenSeqs.splice(lowerBoundSeq(entry.gardenSeqs, seq), 0, seq) + entry.gardenOn.set(seq, record.on) + } else if (type === 'garden-state') { + if (entry.gardenStateSeqs === null) entry.gardenStateSeqs = [] + entry.gardenStateSeqs.splice(lowerBoundSeq(entry.gardenStateSeqs, seq), 0, seq) + } else if (type === 'garden-help') { + if (ownerEvict !== null) dropWriterHelps(ownerEvict, record.to) + if (helpEvict !== null) { + dropHelp(entry, author, helpEvict) + releaseRecord(entry) + } + addHelp(entry, author, seq, record.to, helpAtMs, record.pot) } else if (type === 'fan') { if (entry.fanSeqs === null) { entry.fanSeqs = new Map() @@ -816,13 +1000,78 @@ function createEngagementIndex ({ maxRecords = DEFAULT_MAX_RECORDS, maxTargets = entry.count++ if (entry.count === authorCap) fullAuthors++ records++ - if (targets.size < targetsBefore) freed() + // A writer evicted at the writers cap frees its slots: a paused diary + // may go on. + if (targets.size < targetsBefore || ownerEvict !== null) freed() return true } catch { return false } } + // A live garden-help of `author` at `seq` for pot index `pot`, under the + // writer and under the addressee. + function addHelp (entry, author, seq, to, atMs, pot) { + if (entry.gardenHelps === null) entry.gardenHelps = new Map() + entry.gardenHelps.set(seq, { to, atMs, pot }) + let writers = helpsByTo.get(to) + if (writers === undefined) { + writers = new Map() + helpsByTo.set(to, writers) + } + let list = writers.get(author) + if (list === undefined) { + list = [] + writers.set(author, list) + } + const row = { seq, atMs, pot } + let at = 0 + while (at < list.length && compareHelps(list[at], row) < 0) at++ + list.splice(at, 0, row) + } + + // The writer of `writers` (helpsByTo of one addressee) whose place at + // GARDEN_WRITERS_PER_OWNER may be taken at now: none of its rows to that + // addressee is listable at now. Of those, the one whose newest row is the + // oldest, a tie by the lower key, or null. At most GARDEN_WRITERS_PER_OWNER + // writers of GARDEN_PAIR_MAX rows each. + function evictableWriter (writers, now) { + let best = null + let bestAt = 0 + for (const [writer, list] of writers) { + if (list.some((r) => gardenListable(r.atMs, now))) continue + const newest = list[0].atMs + if (best === null || newest < bestAt || (newest === bestAt && writer < best)) { + best = writer + bestAt = newest + } + } + return best + } + + // Every row of `writer` to `to` leaves both places and frees its slot. + function dropWriterHelps (writer, to) { + const entry = authors.get(writer) + for (const row of helpsByTo.get(to).get(writer).slice()) { + dropHelp(entry, writer, row.seq) + releaseRecord(entry) + } + } + + // The help leaves both places; the caller releases its slot. + function dropHelp (entry, author, seq) { + const info = entry.gardenHelps.get(seq) + entry.gardenHelps.delete(seq) + if (entry.gardenHelps.size === 0) entry.gardenHelps = null + const writers = helpsByTo.get(info.to) + const list = writers?.get(author) + if (list === undefined) return + const at = list.findIndex((r) => r.seq === seq) + if (at !== -1) list.splice(at, 1) + if (list.length === 0) writers.delete(author) + if (writers.size === 0) helpsByTo.delete(info.to) + } + // The last live operation of `author` on `key` (the end of `ops`, sorted // by seq) decides whether `author` is listed under `key`. function publishFan (author, key, ops) { @@ -1098,6 +1347,30 @@ function createEngagementIndex ({ maxRecords = DEFAULT_MAX_RECORDS, maxTargets = found = true } } + if (entry.gardenSeqs !== null) { + const at = indexOfSeq(entry.gardenSeqs, seq) + if (at !== -1) { + entry.gardenSeqs.splice(at, 1) + entry.gardenOn.delete(seq) + if (entry.gardenSeqs.length === 0) { + entry.gardenSeqs = null + entry.gardenOn = null + } + found = true + } + } + if (entry.gardenStateSeqs !== null) { + const at = indexOfSeq(entry.gardenStateSeqs, seq) + if (at !== -1) { + entry.gardenStateSeqs.splice(at, 1) + if (entry.gardenStateSeqs.length === 0) entry.gardenStateSeqs = null + found = true + } + } + if (entry.gardenHelps?.has(seq)) { + dropHelp(entry, author, seq) + found = true + } for (const list of [entry.profileSeqs, entry.usernameSeqs, entry.copiedSeqs, entry.aboutSeqs, entry.moodSeqs]) { const at = indexOfSeq(list, seq) if (at === -1) continue @@ -1242,6 +1515,12 @@ function createEngagementIndex ({ maxRecords = DEFAULT_MAX_RECORDS, maxTargets = listed.delete(author) if (listed.size === 0) pollIn.delete(ref) } + for (const info of entry?.gardenHelps?.values() || EMPTY_LIST) { + const writers = helpsByTo.get(info.to) + if (writers === undefined) continue + writers.delete(author) + if (writers.size === 0) helpsByTo.delete(info.to) + } authors.delete(author) if (entry !== undefined) { if (entry.count === authorCap) fullAuthors-- @@ -1263,19 +1542,20 @@ function createEngagementIndex ({ maxRecords = DEFAULT_MAX_RECORDS, maxTargets = return { profileSeq: entry.profileSeqs.at(-1) ?? null, usernameSeq: entry.usernameSeqs.at(-1) ?? null } } - // Perfil rico pointers of `author`: { about, mood, top, visits }, the seq of + // Perfil rico pointers of `author`: { about, mood, top, visits, garden }, the seq of // its newest live `profile-about` and `mood` block, null when it has none, // top { seq, keys } its newest live `top-friends` block (keys a fresh copy) // or null, and visits the seq of its newest live `visits` block (Visitas: // the pointer says nothing about on or off, the node peeks the block and - // reads it) or null. The object is OPEN: later features add pointers to it and - // readers ignore keys they do not know. The node reads and verifies the - // blocks. + // reads it) or null, and garden the seq of its newest live `garden` block + // (Varandinha, on or off like visits) or null. The object is OPEN: later + // features add pointers to it and readers ignore keys they do not know. The + // node reads and verifies the blocks. function profileExtra (author) { const entry = typeof author === 'string' ? authors.get(author) : undefined const topSeq = entry?.topSeqs?.at(-1) const top = topSeq === undefined ? null : { seq: topSeq, keys: entry.tops.get(topSeq).slice() } - return { about: entry?.aboutSeqs.at(-1) ?? null, mood: entry?.moodSeqs.at(-1) ?? null, top, visits: entry?.visitSeqs?.at(-1) ?? null } + return { about: entry?.aboutSeqs.at(-1) ?? null, mood: entry?.moodSeqs.at(-1) ?? null, top, visits: entry?.visitSeqs?.at(-1) ?? null, garden: entry?.gardenSeqs?.at(-1) ?? null } } // The writers whose last live `fan` operation on `author` is on, newest @@ -2030,6 +2310,156 @@ function createEngagementIndex ({ maxRecords = DEFAULT_MAX_RECORDS, maxTargets = return out } + // The Varandinha pointers of `owner`'s balcony: { on: the seq of its + // newest live `garden` block (on or off, the node peeks it) or null, state: + // its newest live `garden-state` seq or null, helps: [{ author, seq, + // followSeq }], given: [seq], hidden }. A balcony whose newest live + // `garden` block is off, or that has none, answers no state, helps or + // given at once: a member who opted out is not one lookup away from the + // list of who watered them, and no reader shows that balcony anyway. + // helps are the rows of the writers `visible` keeps whose + // followSeq(writer) (the seq of that writer's live follow of the owner) is + // a seq, a throw or null omitting the writer. hidden says a writer with a + // listable row was omitted by `visible`, or the writers cap refused a help + // that may still be listable (gardenFull: a refused help is never + // replayed). A row is listable at now (gardenListable), one per writer + // per Brasilia day (dayStart, the lowest stored seq of that day). The + // reader counts a help only for a pot planted on or before it, so the + // surviving rows are picked per pot (the pot of the day's row): at most + // GARDEN_HELP_ROWS / GARDEN_POTS rows each, and helps naming one pot, or + // an empty one, never push out the helps of another. Inside a pot every + // writer's first row is taken first (the reader keeps one helper row per + // writer), then that pot's other rows, in gardenRowOrder(now) (rows dated + // after now last); the answer is the six picks in that order. given is the owner's own live help seqs inside the same window, newest + // first, at most GARDEN_HELP_ROWS. Null for a malformed owner or clock. + function garden (owner, now, opts = {}) { + try { + const { visible = allVisible, followSeq = noFollow } = opts + if (typeof owner !== 'string' || !HEX64.test(owner)) return null + if (typeof now !== 'number' || !Number.isFinite(now)) return null + const { GARDEN_HELP_ROWS, GARDEN_POTS, dayStart } = gardenRules + const perPot = Math.floor(GARDEN_HELP_ROWS / GARDEN_POTS) + const entry = authors.get(owner) + const on = entry?.gardenSeqs?.at(-1) ?? null + if (on === null || entry.gardenOn.get(on) !== true) return { on, state: null, helps: [], given: [], hidden: false } + const order = gardenRowOrder(now) + let hidden = now < (gardenFull.get(owner) ?? -Infinity) + const byPot = new Map() + for (const [writer, list] of helpsByTo.get(owner) || EMPTY_LIST) { + if (writer === owner) continue + const byDay = new Map() + for (const row of list) { + if (!gardenListable(row.atMs, now)) continue + const day = dayStart(row.atMs) + const prev = byDay.get(day) + if (prev === undefined || row.seq < prev.seq) byDay.set(day, row) + } + if (byDay.size === 0) continue + let shown = false + try { shown = visible(writer) === true } catch {} + if (!shown) { + hidden = true + continue + } + let fseq = null + try { + const n = followSeq(writer) + fseq = isSeq(n) ? n : null + } catch {} + if (fseq === null) continue + const rows = [...byDay.values()].map((r) => ({ atMs: r.atMs, author: writer, seq: r.seq, pot: r.pot, followSeq: fseq })) + rows.sort(order) + const firstIn = new Set() + for (const row of rows) { + let bucket = byPot.get(row.pot) + if (bucket === undefined) { + bucket = { firsts: [], rest: [] } + byPot.set(row.pot, bucket) + } + if (firstIn.has(row.pot)) { + bucket.rest.push(row) + } else { + firstIn.add(row.pot) + bucket.firsts.push(row) + } + } + } + const picked = [] + for (const { firsts, rest } of byPot.values()) { + firsts.sort(order) + rest.sort(order) + const take = firsts.slice(0, perPot) + for (const row of rest) { + if (take.length >= perPot) break + take.push(row) + } + picked.push(...take) + } + picked.sort(order) + const own = [] + for (const [seq, info] of entry.gardenHelps || EMPTY_LIST) { + if (info.to !== owner && gardenListable(info.atMs, now)) own.push({ seq, atMs: info.atMs }) + } + own.sort(compareHelps) + return { + on, + state: entry.gardenStateSeqs?.at(-1) ?? null, + helps: picked.map((r) => ({ author: r.author, seq: r.seq, followSeq: r.followSeq })), + given: own.slice(0, GARDEN_HELP_ROWS).map((r) => r.seq), + hidden + } + } catch { + return null + } + } + + // A Rua das varandinhas: { rows: [{ key, on, state }], next }. Walks the + // keys of `followed` ascending, strictly after `cursor`, and keeps those + // `visible` keeps (a throw hides one) whose newest live `garden` block is + // on: on is that block's seq, state the newest live `garden-state` seq or + // null. visible is asked first for every key examined, and a key it hides + // is skipped without counting: `next` is a public cursor, and a hidden key + // there would name a follow the public follows index omits. At most + // GARDEN_STREET_PAGE rows and GARDEN_STREET_SCAN visible keys per call; + // next is the last visible key examined while keys remain, else null, so + // a page of closed balconies is never a dead end. Null for a malformed + // owner, cursor or `followed`. + function gardenStreet (owner, followed, opts = {}) { + try { + const { visible = allVisible, cursor = null } = opts + if (typeof owner !== 'string' || !HEX64.test(owner)) return null + if (cursor !== null && (typeof cursor !== 'string' || !HEX64.test(cursor))) return null + const { GARDEN_STREET_PAGE } = gardenRules + const keys = new Set() + for (const key of followed) { + if (typeof key === 'string' && HEX64.test(key) && (cursor === null || key > cursor)) keys.add(key) + } + const sorted = [...keys].sort() + const rows = [] + let next = null + let examined = 0 + for (let i = 0; i < sorted.length; i++) { + const key = sorted[i] + let shown = false + try { shown = visible(key) === true } catch {} + if (!shown) continue + examined++ + const entry = authors.get(key) + const on = entry?.gardenSeqs?.at(-1) + if (on !== undefined && entry.gardenOn.get(on) === true) { + rows.push({ key, on, state: entry.gardenStateSeqs?.at(-1) ?? null }) + } + if ((rows.length >= GARDEN_STREET_PAGE || examined >= GARDEN_STREET_SCAN) && i + 1 < sorted.length) { + next = key + break + } + } + return { rows, next } + } catch { + return null + } + } + function stats () { return { records, targets: targets.size, saturated: records >= cap || targets.size >= targetCap || authorCap === 0 || fullAuthors > 0, maxRecords: cap } } @@ -2045,11 +2475,13 @@ function createEngagementIndex ({ maxRecords = DEFAULT_MAX_RECORDS, maxTargets = scrapIn.clear() fanIn.clear() pollIn.clear() + helpsByTo.clear() + gardenFull.clear() records = 0 fullAuthors = 0 } - return { add, removePost, removeRecord, removeDiary, post, roda, counts, engagers, member, hints, noticeRows, noticePage, testimonials, scraps, profileExtra, fans, fanCount, pollTotals, pollVoters, events, stats, clear } + return { add, removePost, removeRecord, removeDiary, post, roda, counts, engagers, member, hints, noticeRows, noticePage, testimonials, scraps, profileExtra, fans, fanCount, pollTotals, pollVoters, events, stats, clear, garden, gardenStreet } } module.exports = { @@ -2079,5 +2511,9 @@ module.exports = { CHOICE_POLLS_MAX, EVENT_PAGE, CHOICE_VOTERS_MAX, + GARDEN_PAIR_MAX, + GARDEN_WRITER_MAX, + GARDEN_WRITERS_PER_OWNER, + GARDEN_STREET_SCAN, CURSOR_RE } diff --git a/packages/core/lib/notices.js b/packages/core/lib/notices.js index b17598ae..647dfc28 100644 --- a/packages/core/lib/notices.js +++ b/packages/core/lib/notices.js @@ -161,7 +161,7 @@ function createNoticeIndex ({ maxPerAuthor = MAX_FOLLOW_TARGETS_PER_AUTHOR, maxE // Exact number of fold entries over all authors (private: never returned). let edges = 0 - return { markPending, foldFollow, commit, removeAuthor, page, rowsFor, followsPage, followCounts, followSeq, followTargets } + return { markPending, foldFollow, commit, removeAuthor, page, rowsFor, followsPage, followCounts, followSeq, followTargets, followersIncomplete, followingIncomplete } function markPending (authorHex) { if (!isHex64(authorHex)) return diff --git a/packages/core/lib/seeder.js b/packages/core/lib/seeder.js index e9d59bcd..c2fc32f7 100644 --- a/packages/core/lib/seeder.js +++ b/packages/core/lib/seeder.js @@ -322,7 +322,7 @@ const FEIRA_OFFERS_PER_DIARY = 4096 const FEIRA_RESCAN_MS = 60000 const FEIRA_RESCAN_MAX_MS = 3600000 const FEIRA_TYPES = new Set(['listing', 'listing-edit', 'listing-status', 'listing-rate']) -const INDEX_TYPES = new Set(['post', 'like', 'repost', 'profile', 'username', 'page-roster', 'page-close', 'page-moved', 'testimonial', 'testimonial-show', 'scrap', 'scrap-hide', 'scrap-wall', 'profile-about', 'mood', 'fan', 'top-friends', 'choice', 'visits']) +const INDEX_TYPES = new Set(['post', 'like', 'repost', 'profile', 'username', 'page-roster', 'page-close', 'page-moved', 'testimonial', 'testimonial-show', 'scrap', 'scrap-hide', 'scrap-wall', 'profile-about', 'mood', 'fan', 'top-friends', 'choice', 'visits', 'garden', 'garden-state', 'garden-help']) // Frame guard for external streams (acceptStream): see lib/frame-guard.js // for the limits, how they were measured, the frame deadlines and the @@ -5340,7 +5340,7 @@ class Seeder { // A closed or moved page leaves the Feira at once. if (isPage && this._names.endSeq(key) !== null && this._feira) this._feiraClose(key) // Page records only feed the name index's hint. - if (record !== null && !isPage && !this._index.add(key, i, record, removed ? { removed: true } : null)) { + if (record !== null && !isPage && !this._index.add(key, i, record, removed ? { removed: true, now: Date.now() } : { now: Date.now() })) { entry.indexPaused = true return } @@ -6887,13 +6887,82 @@ class Seeder { } } + // GET /index/garden/: pointers to the owner's Varandinha, { on, + // state, helps: [{ author, seq, followSeq }], given: [seq], indexing, + // complete } (engagement-index.js garden): on and state are the seqs of + // the newest live `garden` (on or off: the node peeks it) and + // `garden-state` blocks or null; helps are the neighbours' waterings + // inside the reader window of now, from visible writers only, each with + // followSeq, the seq of that HELPER's committed live follow of the owner + // (a help from someone who does not follow the owner counts for nothing, + // so it is left out); given are the owner's own help seqs. A balcony that + // is off, or was never opened, answers state null and no helps or given. + // A held or hidden writer is left out and the answer says complete false, + // as it does while the writers cap refused a help that may still be + // listed (garden hidden) and while the follow index may miss a follow of + // the owner (followersIncomplete: a pending or capped helper has no + // followSeq, so its help is left out); a throw there reads incomplete. Null + // when the owner is not visible or held (_searchHeld), never an empty + // balcony. Fails closed while the owner's own diary is not fully indexed + // and scanned (its newest blocks and tombstones may be missing): nothing + // listed, indexing true, complete false. Keys and seqs only: no pot, + // plant, date or name, which the node reads from the diaries and verifies. + indexGarden (ownerHex) { + try { + if (!this._indexVisible(ownerHex) || this._searchHeld(ownerHex)) return null + const v = this._followsVisible() + const g = this._index.garden(ownerHex, Date.now(), { + visible: v.visible, + followSeq: (hex) => this._notices.followSeq(hex, ownerHex) + }) + if (g === null) return null + if (this._ownerIndexPending(ownerHex)) return { on: null, state: null, helps: [], given: [], indexing: true, complete: false } + const flags = this._indexFlags() + let followsComplete = false + try { followsComplete = this._notices.followersIncomplete(ownerHex) === false } catch {} + return { on: g.on, state: g.state, helps: g.helps, given: g.given, indexing: flags.indexing, complete: flags.complete && !g.hidden && !v.held && followsComplete } + } catch { + return null + } + } + + // GET /index/garden//rua[/]: Varandas da rua, the + // balconies of the accounts the owner follows that show theirs, { items: + // [{ key, on, state }], next, indexing, complete } (engagement-index.js + // gardenStreet): on is the seq of the key's newest live `garden` block, + // which is on, state its newest `garden-state` seq or null; keys + // ascending, strictly after the cursor (a key), `next` the cursor of the + // following page or null. Visible keys only; a held one is left out and + // the answer says complete false, as it does while the owner's own follows + // are pending, truncated or capped (followingIncomplete, a throw reading + // incomplete), since the street may be short. Null when the owner is not visible or + // held or the cursor is malformed. Fails closed while the owner's own + // diary is not fully indexed and scanned (an unfollow may be missing): no + // items, complete false. Keys and seqs only. + indexGardenStreet (ownerHex, cursor = null) { + try { + if (!this._indexVisible(ownerHex) || this._searchHeld(ownerHex)) return null + const v = this._followsVisible() + const page = this._index.gardenStreet(ownerHex, this._notices.followTargets(ownerHex), { visible: v.visible, cursor }) + if (page === null) return null + const flags = this._indexFlags() + if (this._ownerIndexPending(ownerHex)) return { items: [], next: null, indexing: flags.indexing, complete: false } + let followsComplete = false + try { followsComplete = this._notices.followingIncomplete(ownerHex) === false } catch {} + return { items: page.rows, next: page.next, indexing: flags.indexing, complete: flags.complete && !v.held && followsComplete } + } catch { + return null + } + } + // GET /index/profile/: pointers to the author's perfil rico blocks, // { about: seq | null, mood: seq | null, fans: { count, indexing }, top: // { seq, back: { : followSeq | null } } | null, visits: seq | null, - // indexing, complete } (engagement-index.js profileExtra, fanCount): the - // newest live `profile-about`, `mood`, `top-friends` and `visits` block - // (visits is a pointer only: the node peeks the block and checks on === true, - // an off block still has a seq). fans.count is the number + // garden: seq | null, indexing, complete } (engagement-index.js + // profileExtra, fanCount): the newest live `profile-about`, `mood`, + // `top-friends`, `visits` and `garden` block (visits and garden are + // pointers only: the node peeks the block and checks on === true, an off + // block still has a seq). fans.count is the number // of visible fas indexed so far (held ones, and writers whose own diary // is pending, left out: _fansShown), a claim like the follower counts; // fans.indexing is true while it may still grow (any diary not fully @@ -6904,8 +6973,8 @@ class Seeder { // readers ignore keys they do not know. Null when the author is not // visible or held (_searchHeld). Fails closed while the author's own diary // is not fully indexed and scanned (the newest block may be missing, a - // tombstone not yet applied): about, mood, top and visits null, fans.indexing, - // indexing true and complete false. Seqs only: the node reads and verifies + // tombstone not yet applied): about, mood, top, visits and garden null, + // fans.indexing, indexing true and complete false. Seqs only: the node reads and verifies // the blocks. indexProfile (authorHex) { try { @@ -6913,8 +6982,8 @@ class Seeder { const v = this._followsVisible() const shown = this._fansShown(v) const count = this._index.fanCount(authorHex, shown.visible) - if (this._ownerIndexPending(authorHex)) return { about: null, mood: null, fans: { count, indexing: true }, top: null, visits: null, indexing: true, complete: false } - const { about, mood, top, visits } = this._index.profileExtra(authorHex) + if (this._ownerIndexPending(authorHex)) return { about: null, mood: null, fans: { count, indexing: true }, top: null, visits: null, garden: null, indexing: true, complete: false } + const { about, mood, top, visits, garden } = this._index.profileExtra(authorHex) let pointers = null if (top !== null) { const back = {} @@ -6925,7 +6994,7 @@ class Seeder { pointers = { seq: top.seq, back } } const flags = this._indexFlags() - return { about, mood, fans: { count, indexing: !flags.complete || v.held || shown.left() }, top: pointers, visits, indexing: flags.indexing, complete: true } + return { about, mood, fans: { count, indexing: !flags.complete || v.held || shown.left() }, top: pointers, visits, garden, indexing: flags.indexing, complete: true } } catch { return null } diff --git a/packages/core/test/engagement.test.js b/packages/core/test/engagement.test.js index 1f042f34..9a56fe2e 100644 --- a/packages/core/test/engagement.test.js +++ b/packages/core/test/engagement.test.js @@ -223,7 +223,7 @@ test('noticeRows: total on bad input and the index surface is unchanged', (t) => for (const [a, n] of [[null, NOW], ['x', NOW], [OWN.toUpperCase(), NOW], [OWN, NaN], [OWN, '1'], [K2, NOW]]) { t.alike(ix.noticeRows(a, n), []) } - t.alike(Object.keys(ix), ['add', 'removePost', 'removeRecord', 'removeDiary', 'post', 'roda', 'counts', 'engagers', 'member', 'hints', 'noticeRows', 'noticePage', 'testimonials', 'scraps', 'profileExtra', 'fans', 'fanCount', 'pollTotals', 'pollVoters', 'events', 'stats', 'clear']) + t.alike(Object.keys(ix), ['add', 'removePost', 'removeRecord', 'removeDiary', 'post', 'roda', 'counts', 'engagers', 'member', 'hints', 'noticeRows', 'noticePage', 'testimonials', 'scraps', 'profileExtra', 'fans', 'fanCount', 'pollTotals', 'pollVoters', 'events', 'stats', 'clear', 'garden', 'gardenStreet']) }) test('noticeRows: hidden engagers do not take cap slots', (t) => { diff --git a/packages/core/test/garden-index.test.js b/packages/core/test/garden-index.test.js new file mode 100644 index 00000000..e3ab98fb --- /dev/null +++ b/packages/core/test/garden-index.test.js @@ -0,0 +1,808 @@ +'use strict' + +// The Varandinha pointers of the seeder's engagement index (garden(), +// gardenStreet()): latest-wins flags and states, the help rows (follow +// evidence, visibility, one per writer per Brasilia day, the reader window, +// writer fairness), every cap, deletion, purge and paging. No network. +const test = require('brittle') +const { garden: G } = require('@varanda/rpc') +const { + createEngagementIndex, GARDEN_PAIR_MAX, GARDEN_WRITER_MAX, GARDEN_WRITERS_PER_OWNER, GARDEN_STREET_SCAN +} = require('../lib/engagement-index') +const { gardenRecord, gardenStateRecord, gardenHelpRecord, validRecordCopy } = require('../lib/records') + +const O = 'f'.repeat(64) +const W = 'e'.repeat(64) +const HOUR = 3600000 +const DAY = 86400000 +// hh:00 Brasilia time (UTC-3) on 2026-03-. +const bra = (day, hh) => Date.parse('2026-03-' + String(day).padStart(2, '0') + 'T00:00:00.000Z') + 3 * HOUR + hh * HOUR +const NOW = bra(10, 12) +const iso = (ms) => new Date(ms).toISOString() +const k = (i) => (i + 1).toString(16).padStart(64, '0') +const EMPTY_POTS = [null, null, null, null, null, null] +const follows = () => 1 +const open = { followSeq: follows } + +function add (ix, author, seq, record) { + const copy = validRecordCopy(record, seq) + if (copy === null) throw new Error('fixture record is invalid') + return ix.add(author, seq, copy) +} + +// An on balcony for `owner` (garden() lists nothing for an off or unopened +// one), at a seq no fixture help uses. It holds one record slot. +const OPEN_SEQ = 1000000 +function openBalcony (ix, owner, seq = OPEN_SEQ) { + if (!add(ix, owner, seq, gardenRecord({ on: true, at: iso(NOW - DAY) }))) throw new Error('fixture balcony refused') +} + +function help (to, atMs, pot = 0) { + return gardenHelpRecord({ to, pot, at: iso(atMs) }) +} + +function seqs (rows) { + return rows.map((r) => r.author + ':' + r.seq) +} + +test('garden: latest-wins flag and state, on or off, deletes fall back', (t) => { + const ix = createEngagementIndex() + t.alike(ix.garden(O, NOW), { on: null, state: null, helps: [], given: [], hidden: false }) + t.ok(add(ix, O, 1, gardenRecord({ on: true, at: iso(NOW - 5 * HOUR) }))) + t.ok(add(ix, O, 2, gardenStateRecord({ pots: EMPTY_POTS, n: 1, at: iso(NOW - 4 * HOUR) }))) + t.is(ix.garden(O, NOW).on, 1) + t.is(ix.garden(O, NOW).state, 2) + t.ok(add(ix, O, 3, gardenRecord({ on: false, at: iso(NOW - 3 * HOUR) }))) + t.ok(add(ix, O, 4, gardenStateRecord({ pots: EMPTY_POTS, n: 2, at: iso(NOW - 2 * HOUR) }))) + t.is(ix.garden(O, NOW).on, 3, 'an off block is the pointer too') + t.alike(ix.garden(O, NOW), { on: 3, state: null, helps: [], given: [], hidden: false }, 'an off balcony answers no state') + t.alike(ix.gardenStreet(W, [O]).rows, [], 'an off balcony is not on the street') + t.ok(add(ix, O, 5, gardenRecord({ on: true, at: iso(NOW - HOUR) }))) + t.alike(ix.gardenStreet(W, [O]).rows, [{ key: O, on: 5, state: 4 }]) + t.is(ix.stats().records, 5, 'every block holds one slot') + t.ok(ix.removeRecord(O, 5)) + t.is(ix.garden(O, NOW).on, 3, 'a tombstone falls back to the previous block') + t.alike(ix.gardenStreet(W, [O]).rows, [], 'and the street follows the fallback') + t.ok(ix.removeRecord(O, 3)) + t.alike(ix.gardenStreet(W, [O]).rows, [{ key: O, on: 1, state: 4 }]) + t.ok(ix.removeRecord(O, 4)) + t.is(ix.garden(O, NOW).state, 2) + t.ok(add(ix, O, 5, gardenRecord({ on: false, at: iso(NOW) })), 'a tombstoned seq is not resurrected') + t.is(ix.garden(O, NOW).on, 1) + t.ok(ix.removeRecord(O, 1)) + t.ok(ix.removeRecord(O, 2)) + t.alike(ix.garden(O, NOW), { on: null, state: null, helps: [], given: [], hidden: false }) + t.is(ix.stats().records, 0, 'record count back to its start') +}) + +test('garden: a help is listed with its follow evidence; visibility and follow omit writers', (t) => { + const ix = createEngagementIndex() + const A = k(1) + const B = k(2) + openBalcony(ix, O) + t.ok(add(ix, A, 1, help(O, NOW - HOUR, 2))) + t.ok(add(ix, B, 1, help(O, NOW - 2 * HOUR))) + t.alike(ix.garden(O, NOW, { followSeq: (w) => (w === A ? 7 : 9) }).helps, [ + { author: A, seq: 1, followSeq: 7 }, + { author: B, seq: 1, followSeq: 9 } + ]) + const none = ix.garden(O, NOW) + t.alike(none.helps, [], 'no followSeq: every writer omitted') + t.is(none.hidden, false, 'a missing follow is no visibility omission') + const noA = ix.garden(O, NOW, { followSeq: (w) => (w === A ? null : 3) }) + t.alike(seqs(noA.helps), [B + ':1'], 'a writer without followSeq is omitted') + t.is(noA.hidden, false) + const throwsA = ix.garden(O, NOW, { followSeq: (w) => { if (w === A) throw new Error('x'); return 3 } }) + t.alike(seqs(throwsA.helps), [B + ':1'], 'a throwing followSeq omits the writer') + for (const bad of [0, -1, 1.5, '2', NaN]) { + t.alike(ix.garden(O, NOW, { followSeq: () => bad }).helps, [], 'a non-seq followSeq omits: ' + bad) + } + const hidA = ix.garden(O, NOW, { followSeq: follows, visible: (w) => w !== A }) + t.alike(seqs(hidA.helps), [B + ':1']) + t.is(hidA.hidden, true, 'hidden says a writer was omitted for visibility') + const hidThrow = ix.garden(O, NOW, { followSeq: follows, visible: () => { throw new Error('x') } }) + t.alike(hidThrow.helps, []) + t.is(hidThrow.hidden, true, 'a throwing visible hides') + t.alike(ix.garden(W, NOW, open).helps, [], 'another balcony has no rows') + t.is(ix.stats().records, 3) +}) + +test('garden: a self help is never listed, takes no slot and never pauses', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + t.ok(add(ix, O, 1, help(O, NOW))) + t.is(ix.stats().records, 1, 'no slot') + t.alike(ix.garden(O, NOW, open).helps, [], 'never listed') + t.alike(ix.garden(O, NOW, open).given, [], 'never given either') + const full = createEngagementIndex({ maxRecords: 0 }) + t.ok(add(full, O, 1, help(O, NOW)), 'true even with no room') + t.is(add(full, O, 2, help(W, NOW)), false, 'while a real help pauses at the cap') +}) + +test('garden: one row per writer per Brasilia day, the lowest stored seq', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + // 2026-03-10 01:00Z is 22:00 of 2026-03-09 in Brasilia: the same day as 2026-03-09 09:00. + t.ok(add(ix, W, 1, help(O, bra(9, 9)))) + t.ok(add(ix, W, 2, help(O, Date.parse('2026-03-10T01:00:00.000Z')))) + t.is(G.dayStart(bra(9, 9)), G.dayStart(Date.parse('2026-03-10T01:00:00.000Z'))) + t.alike(seqs(ix.garden(O, NOW, open).helps), [W + ':1'], 'one row across UTC midnight, the lowest seq') + t.ok(add(ix, W, 3, help(O, bra(10, 0)))) + t.ok(add(ix, W, 4, help(O, bra(10, 0) - 1))) + t.alike(seqs(ix.garden(O, NOW, open).helps), [W + ':3', W + ':1'], '00:00 Brasilia starts a new day, the seq 4 row joins day 9') + t.ok(ix.removeRecord(W, 1)) + t.alike(seqs(ix.garden(O, NOW, open).helps), [W + ':3', W + ':2'], 'the next stored seq of that day takes over') +}) + +test('garden: a tombstoned help leaves the list and the next of the four is listed', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + for (let s = 1; s <= GARDEN_PAIR_MAX; s++) t.ok(add(ix, W, s, help(O, bra(10, s)))) + t.is(ix.stats().records, GARDEN_PAIR_MAX + 1) + t.alike(seqs(ix.garden(O, NOW, open).helps), [W + ':1']) + for (let s = 1; s < GARDEN_PAIR_MAX; s++) { + t.ok(ix.removeRecord(W, s)) + t.alike(seqs(ix.garden(O, NOW, open).helps), [W + ':' + (s + 1)], 'seq ' + (s + 1) + ' listed after ' + s + ' is deleted') + } + t.ok(ix.removeRecord(W, GARDEN_PAIR_MAX)) + t.alike(ix.garden(O, NOW, open).helps, []) + t.is(ix.stats().records, 1, 'every row freed its slot') +}) + +test('garden: the reader window and the future cut', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + const cases = [ + [NOW - G.GARDEN_HELP_WINDOW_MS, true], + [NOW - G.GARDEN_HELP_WINDOW_MS - 1, false], + [NOW + G.GARDEN_MAX_FUTURE_MS, true], + [NOW + G.GARDEN_MAX_FUTURE_MS + 1, false], + [NOW, true] + ] + cases.forEach(([atMs], i) => t.ok(add(ix, k(i), 1, help(O, atMs)))) + const listed = new Set(ix.garden(O, NOW, open).helps.map((r) => r.author)) + cases.forEach(([atMs, want], i) => t.is(listed.has(k(i)), want, iso(atMs))) + t.is(ix.stats().records, cases.length + 1, 'rows out of the window still hold their slot') + t.is(ix.garden(O, NOW + 2 * DAY, open).helps.length, 3, 'a later clock lists the once future rows and drops the old ones') +}) + +test('garden: given lists the own live helps in the window, newest first, capped', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + t.ok(add(ix, O, 1, help(k(1), NOW - 2 * HOUR))) + t.ok(add(ix, O, 2, help(k(2), NOW - HOUR))) + t.ok(add(ix, O, 3, help(k(3), NOW - G.GARDEN_HELP_WINDOW_MS - 1))) + t.ok(add(ix, O, 4, help(k(4), NOW + G.GARDEN_MAX_FUTURE_MS + 1))) + t.alike(ix.garden(O, NOW).given, [2, 1]) + t.ok(ix.removeRecord(O, 2)) + t.alike(ix.garden(O, NOW).given, [1], 'a deleted help leaves given') + const many = createEngagementIndex() + openBalcony(many, O) + for (let i = 0; i < G.GARDEN_HELP_ROWS + 6; i++) t.ok(add(many, O, i + 1, help(k(i), NOW - (i + 1) * 60000))) + const given = many.garden(O, NOW).given + t.is(given.length, G.GARDEN_HELP_ROWS) + t.is(given[0], 1, 'newest first') +}) + +test('garden: every writer is listed before any writer gets a second day', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + const busy = [k(100), k(101), k(102), k(103), k(104)] + // Five days inside the window, newest first: the busy writers keep their + // four latest seqs. + const days = [bra(11, 6), bra(10, 6), bra(9, 6), bra(8, 6), bra(7, 18)] + // Writers spread over the six pots (writer i to pot i % 6): pots 0 to 4 + // hold one busy writer each, so each lists 4 rows, and pot 5 only its two + // quiet writers. + const potOf = (w) => [...busy, ...quiet].indexOf(w) % G.GARDEN_POTS + const quiet = [] + for (let i = 0; i < 10; i++) quiet.push(k(200 + i)) + for (const w of busy) days.forEach((atMs, i) => t.ok(add(ix, w, i + 1, help(O, atMs, potOf(w))))) + for (const w of quiet) t.ok(add(ix, w, 1, help(O, bra(7, 19), potOf(w)))) + t.is(ix.stats().records, 1 + busy.length * GARDEN_PAIR_MAX + quiet.length, 'the lowest seq busy row was dropped and freed') + const helps = ix.garden(O, NOW, open).helps + t.is(helps.length, 5 * 4 + 2) + const authors = new Set(helps.map((r) => r.author)) + for (const w of [...busy, ...quiet]) t.ok(authors.has(w), 'listed: ' + w.slice(-3)) + const planted = { plant: 'tomate', at: iso(bra(1, 0)), water: iso(bra(1, 0)), w: 0 } + const fold = G.foldHelps({ + owner: O, + now: NOW, + pots: new Array(G.GARDEN_POTS).fill(planted), + helps: helps.map((r) => ({ author: r.author, seq: r.seq, record: help(O, rowAt(r), potOf(r.author)), follow: { type: 'follow', key: O, on: true, at: iso(bra(1, 0)) } })) + }) + t.is(fold.helpers.length, busy.length + quiet.length, 'the reader fold sees every helper') + + function rowAt (r) { + return busy.includes(r.author) ? days[r.seq - 1] : bra(7, 19) + } +}) + +test('garden: per pair cap keeps the latest listable seqs, an unlistable incoming row is skipped without a slot', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + const base = ix.stats().records + // Days 10, 9, 9 and 8 in Brasilia: seq 3 repeats day 9 and is never listed. + for (let s = 1; s <= GARDEN_PAIR_MAX; s++) t.ok(add(ix, W, s, help(O, NOW - s * DAY / 2))) + t.is(ix.stats().records - base, GARDEN_PAIR_MAX) + t.alike(seqs(ix.garden(O, NOW, open).helps), [W + ':1', W + ':2', W + ':4']) + t.ok(add(ix, W, 10, help(O, NOW - 2 * DAY - 1)), 'a second day 8 row: true') + t.is(ix.stats().records - base, GARDEN_PAIR_MAX, 'and no slot') + t.ok(add(ix, W, 11, help(O, bra(7, 18))), 'a new day') + t.is(ix.stats().records - base, GARDEN_PAIR_MAX, 'evicts the unlistable seq 3 and frees its slot') + t.alike(seqs(ix.garden(O, NOW, open).helps), [W + ':1', W + ':2', W + ':4', W + ':11']) + t.is(ix.removeRecord(W, 3), false, 'the evicted seq is no longer indexed') + t.is(ix.stats().records - base, GARDEN_PAIR_MAX, 'and its delete frees nothing more') + t.ok(add(ix, W, 12, help(O, bra(11, 6))), 'another new day, every kept row listable') + t.is(ix.stats().records - base, GARDEN_PAIR_MAX, 'the lowest seq leaves') + t.alike(seqs(ix.garden(O, NOW, open).helps), [W + ':2', W + ':4', W + ':11', W + ':12'], 'day 10 lost seq 1, the row dated after now listed last') + t.is(ix.removeRecord(W, 1), false) + t.ok(ix.removeRecord(W, 12)) + t.is(ix.stats().records - base, GARDEN_PAIR_MAX - 1) +}) + +test('garden: per writer cap of help rows, the lowest seq dropped and freed', (t) => { + const ix = createEngagementIndex() + for (const owner of [k(0), k(1), k(500), k(501), W]) openBalcony(ix, owner) + const base = ix.stats().records + for (let i = 0; i < GARDEN_WRITER_MAX; i++) t.ok(add(ix, W, i + 1, help(k(i), NOW - (GARDEN_WRITER_MAX - i) * 60000))) + t.is(ix.stats().records - base, GARDEN_WRITER_MAX) + t.alike(seqs(ix.garden(k(0), NOW, open).helps), [W + ':1']) + t.ok(add(ix, W, 100, help(k(500), NOW - DAY)), 'an older date still takes a slot') + t.is(ix.stats().records - base, GARDEN_WRITER_MAX, 'the lowest seq left and freed its slot') + t.alike(ix.garden(k(0), NOW, open).helps, [], 'the lowest seq balcony lost its row') + t.alike(seqs(ix.garden(k(500), NOW, open).helps), [W + ':100']) + t.ok(add(ix, W, 101, help(k(501), NOW))) + t.is(ix.stats().records - base, GARDEN_WRITER_MAX) + t.alike(ix.garden(k(1), NOW, open).helps, [], 'then the next lowest seq') + t.alike(seqs(ix.garden(k(501), NOW, open).helps), [W + ':101']) + t.is(ix.garden(W, NOW).given.length, G.GARDEN_HELP_ROWS) +}) + +test('garden: far-future helps never cost an in-window help its pair slot', (t) => { + const far = Date.parse('9999-12-31T00:00:00.000Z') + const late = GARDEN_PAIR_MAX + 1 + const same = createEngagementIndex() + openBalcony(same, O) + for (let s = 1; s <= GARDEN_PAIR_MAX; s++) t.ok(add(same, W, s, help(O, far))) + t.ok(add(same, W, late, help(O, NOW - HOUR))) + t.alike(seqs(same.garden(O, NOW, open).helps), [W + ':' + late], 'four far-future rows of one day keep no in-window help out') + t.ok(same.removeDiary(W)) + t.is(same.stats().records, 1, 'record count back to its start') + const spread = createEngagementIndex() + openBalcony(spread, O) + for (let s = 1; s <= GARDEN_PAIR_MAX; s++) t.ok(add(spread, W, s, help(O, far - s * DAY))) + t.ok(add(spread, W, late, help(O, NOW - HOUR))) + t.alike(seqs(spread.garden(O, NOW, open).helps), [W + ':' + late], 'four far-future days keep no in-window help out') + t.is(spread.stats().records, GARDEN_PAIR_MAX + 1, 'the lowest seq left and freed its slot') + t.is(spread.removeRecord(W, 1), false) + t.ok(spread.removeDiary(W)) + t.is(spread.stats().records, 1, 'record count back to its start') +}) + +test('garden: far-future helps never cost an in-window help its writer slot', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + const base = ix.stats().records + const far = Date.parse('9999-12-31T00:00:00.000Z') + for (let i = 0; i < GARDEN_WRITER_MAX; i++) t.ok(add(ix, W, i + 1, help(k(i), far))) + t.is(ix.stats().records - base, GARDEN_WRITER_MAX) + t.ok(add(ix, W, GARDEN_WRITER_MAX + 1, help(O, NOW - HOUR))) + t.alike(seqs(ix.garden(O, NOW, open).helps), [W + ':' + (GARDEN_WRITER_MAX + 1)], 'the in-window help is listed') + t.is(ix.stats().records - base, GARDEN_WRITER_MAX) + t.is(ix.removeRecord(W, 1), false, 'the lowest seq left') + t.ok(ix.removeDiary(W)) + t.is(ix.stats().records, base, 'record count back to its start') +}) + +test('garden: past the pair cap a same-day duplicate leaves, never the lowest seq of its day', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + const base = ix.stats().records + t.ok(add(ix, W, 1, help(O, bra(7, 18)))) + t.ok(add(ix, W, 2, help(O, bra(7, 20))), 'a second help on day 7, which an honest node refuses') + t.ok(add(ix, W, 3, help(O, bra(8, 9)))) + t.ok(add(ix, W, 4, help(O, bra(9, 9)))) + t.ok(add(ix, W, 5, help(O, bra(10, 9)))) + t.is(ix.stats().records - base, GARDEN_PAIR_MAX) + t.alike(seqs(ix.garden(O, NOW, open).helps), [W + ':5', W + ':4', W + ':3', W + ':1'], 'day 7 lists seq 1, never seq 2') + t.is(ix.removeRecord(W, 2), false, 'the duplicate seq 2 was the one evicted') + t.ok(ix.removeRecord(W, 1)) + t.alike(seqs(ix.garden(O, NOW, open).helps), [W + ':5', W + ':4', W + ':3'], 'seq 2 is not stored, so day 7 lists nothing') + t.ok(ix.removeDiary(W)) + t.is(ix.stats().records, base, 'record count back to its start') + + const kept = createEngagementIndex() + openBalcony(kept, O) + t.ok(add(kept, W, 1, help(O, bra(7, 18)))) + t.ok(add(kept, W, 2, help(O, bra(7, 20)))) + t.ok(add(kept, W, 3, help(O, bra(8, 9)))) + t.ok(add(kept, W, 4, help(O, bra(9, 9)))) + t.alike(seqs(kept.garden(O, NOW, open).helps), [W + ':4', W + ':3', W + ':1']) + t.ok(kept.removeRecord(W, 1)) + t.alike(seqs(kept.garden(O, NOW, open).helps), [W + ':4', W + ':3', W + ':2'], 'a stored duplicate is listed once the lowest seq of its day is deleted') + t.ok(kept.removeDiary(W)) + t.is(kept.stats().records, 1, 'record count back to its start') +}) + +test('garden: past the writer cap a same-day duplicate leaves, never the lowest seq of any day', (t) => { + const ix = createEngagementIndex() + for (const owner of [O, k(3), k(500)]) openBalcony(ix, owner) + const base = ix.stats().records + t.ok(add(ix, W, 1, help(O, bra(9, 9)))) + t.ok(add(ix, W, 2, help(O, bra(9, 10)))) + for (let s = 3; s <= GARDEN_WRITER_MAX; s++) t.ok(add(ix, W, s, help(k(s), NOW - (GARDEN_WRITER_MAX - s + 1) * 60000))) + t.is(ix.stats().records - base, GARDEN_WRITER_MAX) + t.ok(add(ix, W, GARDEN_WRITER_MAX + 1, help(k(500), NOW))) + t.is(ix.stats().records - base, GARDEN_WRITER_MAX) + t.alike(seqs(ix.garden(O, NOW, open).helps), [W + ':1'], 'day 9 still lists seq 1') + t.is(ix.removeRecord(W, 2), false, 'the duplicate seq 2 was the one evicted') + t.alike(seqs(ix.garden(k(3), NOW, open).helps), [W + ':3'], 'the lowest seq of another balcony stays') + t.alike(seqs(ix.garden(k(500), NOW, open).helps), [W + ':' + (GARDEN_WRITER_MAX + 1)]) + t.ok(ix.removeDiary(W)) + t.is(ix.stats().records, base, 'record count back to its start') +}) + +test('garden: writers per balcony cap, a new writer past it takes no slot', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + const base = ix.stats().records + for (let i = 0; i < GARDEN_WRITERS_PER_OWNER; i++) t.ok(add(ix, k(i), 1, help(O, NOW - DAY, i % G.GARDEN_POTS))) + t.is(ix.stats().records - base, GARDEN_WRITERS_PER_OWNER) + const extra = k(GARDEN_WRITERS_PER_OWNER) + t.ok(add(ix, extra, 1, help(O, NOW)), 'true') + t.is(ix.stats().records - base, GARDEN_WRITERS_PER_OWNER, 'no slot') + t.ok(add(ix, k(0), 2, help(O, NOW)), 'a writer already listed may add another day') + t.is(ix.stats().records - base, GARDEN_WRITERS_PER_OWNER + 1) + const helps = ix.garden(O, NOW, open).helps + t.is(helps.length, G.GARDEN_HELP_ROWS) + t.absent(helps.some((r) => r.author === extra), 'never listed') + t.ok(ix.removeDiary(k(1))) + t.ok(add(ix, extra, 2, help(O, NOW)), 'a freed writer place takes the new writer') + t.is(ix.stats().records - base, GARDEN_WRITERS_PER_OWNER + 1) +}) + +function addAt (ix, author, seq, record, now) { + const copy = validRecordCopy(record, seq) + if (copy === null) throw new Error('fixture record is invalid') + return ix.add(author, seq, copy, { now }) +} + +test('garden: at the writers cap, a writer with no listable row gives its place to a listable help', (t) => { + let freed = 0 + const ix = createEngagementIndex({ onFreed: () => freed++ }) + openBalcony(ix, O) + const base = ix.stats().records + // Every place is held by rows older than the window at now; k(0) holds + // the oldest newest row and two rows. + const stale = (i) => NOW - G.GARDEN_HELP_WINDOW_MS - (GARDEN_WRITERS_PER_OWNER - i) * 60000 + t.ok(addAt(ix, k(0), 1, help(O, stale(0) - 60000), NOW)) + for (let i = 0; i < GARDEN_WRITERS_PER_OWNER; i++) t.ok(addAt(ix, k(i), 2, help(O, stale(i)), NOW)) + t.is(ix.stats().records - base, GARDEN_WRITERS_PER_OWNER + 1) + const extra = k(GARDEN_WRITERS_PER_OWNER) + freed = 0 + t.ok(addAt(ix, extra, 1, help(O, NOW - HOUR), NOW), 'the new writer is stored') + t.alike(seqs(ix.garden(O, NOW, open).helps), [extra + ':1'], 'and listed') + t.is(ix.garden(O, NOW, open).hidden, false) + t.is(ix.removeRecord(k(0), 1), false, 'the evicted writer row is gone') + t.is(ix.removeRecord(k(0), 2), false, 'every row of it') + t.is(ix.stats().records - base, GARDEN_WRITERS_PER_OWNER, 'its two slots freed, one taken') + t.ok(freed > 0, 'the freed slots wake paused diaries') + for (let i = 0; i <= GARDEN_WRITERS_PER_OWNER; i++) t.ok(ix.removeDiary(k(i))) + t.is(ix.stats().records, base, 'record count back to its start') +}) + +test('garden: at the writers cap with every row listable, a refused help hides the balcony until it leaves the window', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + for (let i = 0; i < GARDEN_WRITERS_PER_OWNER; i++) t.ok(addAt(ix, k(i), 1, help(O, NOW - HOUR), NOW)) + const base = ix.stats().records + const extra = k(GARDEN_WRITERS_PER_OWNER) + const refusedAt = NOW - 2 * HOUR + t.ok(addAt(ix, extra, 1, help(O, refusedAt), NOW), 'skipped: true') + t.is(ix.stats().records, base, 'no slot, no eviction') + t.is(ix.garden(O, NOW, open).hidden, true, 'a help it might list was refused') + t.is(ix.garden(O, refusedAt + G.GARDEN_HELP_WINDOW_MS, open).hidden, true, 'still listable at the window edge') + t.is(ix.garden(O, refusedAt + G.GARDEN_HELP_WINDOW_MS + 1, open).hidden, false, 'out of the window: honest again') + t.is(ix.garden(W, NOW, open).hidden, false, 'another balcony is not full') + t.ok(ix.removeDiary(O)) + openBalcony(ix, O) + t.is(ix.garden(O, NOW, open).hidden, true, 'the mark outlives the owner diary, like the rows addressed to it') + ix.clear() + openBalcony(ix, O) + t.is(ix.garden(O, NOW, open).hidden, false, 'clear resets it') +}) + +test('garden: at the writers cap, an incoming help that cannot be listed is skipped and marks nothing', (t) => { + const far = NOW + G.GARDEN_MAX_FUTURE_MS + 1 + const old = NOW - G.GARDEN_HELP_WINDOW_MS - 1 + const full = createEngagementIndex() + openBalcony(full, O) + for (let i = 0; i < GARDEN_WRITERS_PER_OWNER; i++) t.ok(addAt(full, k(i), 1, help(O, NOW - HOUR), NOW)) + const base = full.stats().records + t.ok(addAt(full, k(GARDEN_WRITERS_PER_OWNER), 1, help(O, far), NOW), 'far future: true') + t.ok(addAt(full, k(GARDEN_WRITERS_PER_OWNER + 1), 1, help(O, old), NOW), 'stale: true') + t.is(full.stats().records, base, 'no slot') + t.is(full.garden(O, NOW, open).hidden, false, 'a row that could never be listed hides nothing') + const stale = createEngagementIndex() + openBalcony(stale, O) + for (let i = 0; i < GARDEN_WRITERS_PER_OWNER; i++) t.ok(addAt(stale, k(i), 1, help(O, old - i), NOW)) + const before = stale.stats().records + t.ok(addAt(stale, k(GARDEN_WRITERS_PER_OWNER), 1, help(O, far), NOW)) + t.ok(addAt(stale, k(GARDEN_WRITERS_PER_OWNER + 1), 1, help(O, old), NOW)) + t.is(stale.stats().records, before, 'nor does it evict a writer') + t.is(stale.removeRecord(k(GARDEN_WRITERS_PER_OWNER - 1), 1), true, 'the oldest writer is still there') +}) + +test('garden: at the writers cap without a clock, a new writer is refused and the balcony says hidden', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + for (let i = 0; i < GARDEN_WRITERS_PER_OWNER; i++) t.ok(add(ix, k(i), 1, help(O, NOW - G.GARDEN_HELP_WINDOW_MS - HOUR))) + const base = ix.stats().records + t.ok(add(ix, k(GARDEN_WRITERS_PER_OWNER), 1, help(O, NOW - HOUR)), 'skipped: true') + t.is(ix.stats().records, base, 'no slot and no eviction without a clock') + t.alike(ix.garden(O, NOW, open).helps, []) + t.is(ix.garden(O, NOW, open).hidden, true) +}) + +test('garden: a help that evicts its own writer row is net zero and passes a full budget', (t) => { + for (const opts of [{ perAuthorMax: GARDEN_PAIR_MAX }, { maxRecords: GARDEN_PAIR_MAX + 1 }]) { + const ix = createEngagementIndex(opts) + openBalcony(ix, O) + // Four listable days, then a fifth one: every row is the first of its day. + const days = [bra(9, 18), bra(8, 18), bra(7, 18), bra(10, 1)] + days.forEach((atMs, i) => t.ok(add(ix, W, i + 1, help(O, atMs)))) + const base = ix.stats().records + t.ok(add(ix, W, 10, help(O, bra(11, 6))), 'a fifth help is stored: ' + JSON.stringify(opts)) + t.is(ix.stats().records, base, 'the count is unchanged') + t.is(ix.removeRecord(W, 1), false, 'the lowest seq left') + t.alike(seqs(ix.garden(O, NOW, open).helps), [W + ':4', W + ':2', W + ':3', W + ':10'], 'the new row replaced it') + if (opts.perAuthorMax) t.is(add(ix, W, 11, help(k(9), NOW)), false, 'a help that evicts nothing still pauses at the author cap') + else t.is(add(ix, k(1), 1, help(O, NOW)), false, 'another writer still pauses at the budget') + } +}) + +test('garden: with a clock, the pair cap keeps a day row the window edge left alone', (t) => { + const rows = [bra(7, 1), bra(7, 23), bra(8, 9), bra(9, 9)] + t.is(G.dayStart(rows[0]), G.dayStart(rows[1]), 'seq 1 and 2 share a Brasilia day') + const ix = createEngagementIndex() + openBalcony(ix, O) + rows.forEach((atMs, i) => t.ok(addAt(ix, W, i + 1, help(O, atMs), NOW))) + t.alike(seqs(ix.garden(O, NOW, open).helps), [W + ':4', W + ':3', W + ':2'], 'seq 1 is out of the window, seq 2 lists day 7') + t.ok(addAt(ix, W, 5, help(O, bra(10, 9)), NOW)) + t.is(ix.removeRecord(W, 1), false, 'the out of window seq 1 was evicted') + t.alike(seqs(ix.garden(O, NOW, open).helps), [W + ':5', W + ':4', W + ':3', W + ':2'], 'the in-window seq 2 stays listed') + const old = createEngagementIndex() + openBalcony(old, O) + rows.forEach((atMs, i) => t.ok(add(old, W, i + 1, help(O, atMs)))) + t.ok(add(old, W, 5, help(O, bra(10, 9)))) + t.is(old.removeRecord(W, 2), false, 'without a clock the same-day duplicate seq 2 leaves, as before') +}) + +test('garden: helps dated after the seeder clock rank after every honest one', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + for (let i = 0; i < G.GARDEN_HELP_ROWS; i++) t.ok(add(ix, k(i), 1, help(O, NOW + 23 * HOUR, i % G.GARDEN_POTS))) + const honest = k(500) + t.ok(add(ix, honest, 1, help(O, NOW - HOUR))) + const helps = ix.garden(O, NOW, open).helps + t.is(helps.length, G.GARDEN_HELP_ROWS) + t.is(helps[0].author, honest, 'the honest help is listed, first') + t.ok(add(ix, k(600), 1, help(O, NOW + HOUR))) + t.is(ix.garden(O, NOW, open).helps[1].author, k(600), 'among future rows, the nearest to now first') +}) + +// The reader fold counts a help only for a pot planted on or before it, so +// rows naming one pot (an empty one here) must not take the places of +// another pot's honest rows. +test('garden: helps for an unplanted pot never crowd out the honest helps of another pot', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + const start = ix.stats().records + const honest = [k(1), k(2), k(3)] + for (const w of honest) t.ok(add(ix, w, 1, help(O, NOW - 10 * HOUR, 0))) + const flood = [] + for (let i = 0; i < G.GARDEN_HELP_ROWS; i++) { + flood.push(k(100 + i)) + t.ok(add(ix, k(100 + i), 1, help(O, NOW - HOUR, 5))) + } + const helps = ix.garden(O, NOW, open).helps + const authors = helps.map((r) => r.author) + for (const w of honest) t.ok(authors.includes(w), 'the honest help is listed: ' + w.slice(-1)) + t.ok(helps.filter((r) => flood.includes(r.author)).length <= G.GARDEN_HELP_ROWS / G.GARDEN_POTS, 'at most 4 rows name pot 5') + t.is(helps.length, honest.length + G.GARDEN_HELP_ROWS / G.GARDEN_POTS) + const planted = { plant: 'tomate', at: iso(NOW - DAY), water: iso(NOW - DAY), w: 0 } + const fold = G.foldHelps({ + owner: O, + now: NOW, + pots: [planted, null, null, null, null, null], + helps: helps.map((r) => ({ author: r.author, seq: r.seq, record: help(O, honest.includes(r.author) ? NOW - 10 * HOUR : NOW - HOUR, honest.includes(r.author) ? 0 : 5), follow: { type: 'follow', key: O, on: true, at: iso(NOW - DAY) } })) + }) + t.is(fold.perPot[0], G.GARDEN_HELP_PER_POT, 'the reader counts the honest helps') + for (const w of [...honest, ...flood]) t.ok(ix.removeDiary(w)) + t.is(ix.stats().records, start, 'record count back to its start') +}) + +test('garden: at most GARDEN_HELP_ROWS / GARDEN_POTS rows per pot, the last ranked row of each pot left out', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + const perPot = G.GARDEN_HELP_ROWS / G.GARDEN_POTS + const dropped = [] + for (let p = 0; p < G.GARDEN_POTS; p++) { + for (let j = 0; j <= perPot; j++) { + // Pot 0 holds the newest rows, so a global pick of the newest 24 + // would fill pots 0 to 3 and leave pot 5 out. + const w = k(10 * p + j) + t.ok(add(ix, w, 1, help(O, NOW - (p * (perPot + 1) + j + 1) * HOUR, p))) + if (j === perPot) dropped.push(w) + } + } + const helps = ix.garden(O, NOW, open).helps + t.is(helps.length, G.GARDEN_HELP_ROWS) + for (let p = 0; p < G.GARDEN_POTS; p++) { + const inPot = helps.filter((r) => Math.floor((parseInt(r.author, 16) - 1) / 10) === p) + t.is(inPot.length, perPot, 'pot ' + p + ' lists ' + perPot + ' rows') + } + for (const w of dropped) t.absent(helps.some((r) => r.author === w), 'the last ranked row of its pot is left out') + const atOf = (r) => { const n = parseInt(r.author, 16) - 1; return NOW - ((Math.floor(n / 10)) * (perPot + 1) + (n % 10) + 1) * HOUR } + for (let i = 1; i < helps.length; i++) t.ok(atOf(helps[i - 1]) >= atOf(helps[i]), 'the answer is newest first') +}) + +test('garden: inside a pot every writer is listed before a writer gets a second day', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + const busy = k(1) + const days = [bra(10, 11), bra(9, 11), bra(8, 11)] + days.forEach((atMs, i) => t.ok(add(ix, busy, i + 1, help(O, atMs, 2)))) + const quiet = [k(2), k(3), k(4)] + for (const w of quiet) t.ok(add(ix, w, 1, help(O, bra(7, 20), 2))) + const helps = ix.garden(O, NOW, open).helps + t.alike(seqs(helps), [busy + ':1', ...quiet.map((w) => w + ':1')], 'the three other writers before the busy writer\'s second day') +}) + +test('garden: the per writer and day row is chosen across pots, and its pot is the bucket', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + t.ok(add(ix, W, 5, help(O, bra(10, 8), 1))) + t.ok(add(ix, W, 6, help(O, bra(10, 9), 3))) + // Pot 3 is full of newer rows: W's row is listed only if it sits in pot 1. + const others = [k(1), k(2), k(3), k(4)] + for (const w of others) t.ok(add(ix, w, 1, help(O, bra(10, 10), 3))) + const helps = ix.garden(O, NOW, open).helps + t.ok(seqs(helps).includes(W + ':5'), 'the lowest seq of the day is listed, under pot 1') + t.absent(seqs(helps).includes(W + ':6'), 'the later seq of the same day is not') + t.is(helps.length, 1 + others.length) +}) + +test('garden: a row dated after now ranks last inside its pot', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + const ahead = k(100) + t.ok(add(ix, ahead, 1, help(O, NOW + HOUR, 0))) + const past = [k(1), k(2), k(3), k(4)] + for (const w of past) t.ok(add(ix, w, 1, help(O, NOW - 20 * HOUR, 0))) + const later = k(200) + t.ok(add(ix, later, 1, help(O, NOW + HOUR, 1))) + t.ok(add(ix, k(5), 1, help(O, NOW - 30 * HOUR, 1))) + const helps = ix.garden(O, NOW, open).helps + t.absent(helps.some((r) => r.author === ahead), 'the future row is the one its full pot leaves out') + t.is(helps.length, past.length + 2) + t.is(helps.at(-1).author, later, 'a future row in a pot with room is listed, last') +}) + +test('garden: an off or never opened balcony answers no helps, given or state, without a writer walk', (t) => { + for (const off of [true, false]) { + const ix = createEngagementIndex() + if (off) { + t.ok(add(ix, O, 1, gardenRecord({ on: true, at: iso(NOW - 3 * HOUR) }))) + t.ok(add(ix, O, 2, gardenStateRecord({ pots: EMPTY_POTS, n: 1, at: iso(NOW - 2 * HOUR) }))) + t.ok(add(ix, O, 3, gardenRecord({ on: false, at: iso(NOW - HOUR) }))) + } else { + t.ok(add(ix, O, 2, gardenStateRecord({ pots: EMPTY_POTS, n: 1, at: iso(NOW - 2 * HOUR) }))) + } + t.ok(add(ix, O, 4, help(W, NOW - HOUR))) + for (let i = 0; i < 3; i++) t.ok(add(ix, k(i), 1, help(O, NOW - HOUR))) + let calls = 0 + const g = ix.garden(O, NOW, { visible: () => { calls++; return true }, followSeq: () => { calls++; return 1 } }) + t.alike(g, { on: off ? 3 : null, state: null, helps: [], given: [], hidden: false }, off ? 'off' : 'never opened') + t.is(calls, 0, 'visible and followSeq are never called') + } +}) + +test('garden: removeDiary of a writer and of an owner', (t) => { + const ix = createEngagementIndex() + const A = k(1) + openBalcony(ix, A) + t.ok(add(ix, O, 1, gardenRecord({ on: true, at: iso(NOW - DAY) }))) + t.ok(add(ix, O, 2, gardenStateRecord({ pots: EMPTY_POTS, n: 1, at: iso(NOW - DAY) }))) + t.ok(add(ix, O, 3, help(A, NOW - HOUR))) + t.ok(add(ix, A, 1, help(O, NOW - HOUR))) + t.ok(add(ix, W, 1, help(O, NOW - 2 * HOUR))) + t.ok(add(ix, W, 2, help(A, NOW - 2 * HOUR))) + t.is(ix.stats().records, 7) + t.ok(ix.removeDiary(W)) + t.alike(seqs(ix.garden(O, NOW, open).helps), [A + ':1'], 'the writer rows leave every balcony') + t.alike(seqs(ix.garden(A, NOW, open).helps), [O + ':3']) + t.is(ix.stats().records, 5) + t.ok(ix.removeDiary(O)) + t.alike(ix.garden(O, NOW, open), { on: null, state: null, helps: [], given: [], hidden: false }, 'no balcony block: nothing listed') + t.alike(ix.garden(A, NOW, open).helps, [], 'the owner rows to others leave') + t.alike(ix.gardenStreet(W, [O]).rows, []) + t.is(ix.stats().records, 2) + openBalcony(ix, O) + const back = ix.garden(O, NOW, open) + t.is(back.state, null) + t.alike(back.given, []) + t.alike(seqs(back.helps), [A + ':1'], 'rows addressed to the owner belong to their writers and stay') + t.ok(ix.removeDiary(A)) + t.alike(ix.garden(O, NOW, open).helps, []) + t.ok(ix.removeDiary(O)) + t.is(ix.stats().records, 0, 'record count back to its start') + openBalcony(ix, O) + t.ok(add(ix, A, 5, help(O, NOW))) + ix.clear() + t.alike(ix.garden(O, NOW, open).helps, [], 'clear purges the rows') + t.is(ix.stats().records, 0) +}) + +test('garden: a duplicate seq is ignored across the three types', (t) => { + const ix = createEngagementIndex() + openBalcony(ix, O) + const base = ix.stats().records + t.ok(add(ix, W, 1, gardenRecord({ on: true, at: iso(NOW) }))) + t.ok(add(ix, W, 2, gardenStateRecord({ pots: EMPTY_POTS, n: 1, at: iso(NOW) }))) + t.ok(add(ix, W, 3, help(O, NOW))) + t.ok(add(ix, W, 1, gardenRecord({ on: false, at: iso(NOW) }))) + t.ok(add(ix, W, 1, help(O, NOW))) + t.ok(add(ix, W, 2, gardenRecord({ on: false, at: iso(NOW) }))) + t.ok(add(ix, W, 3, help(O, NOW + HOUR))) + t.ok(add(ix, W, 3, gardenStateRecord({ pots: EMPTY_POTS, n: 2, at: iso(NOW) }))) + t.is(ix.stats().records - base, 3) + t.alike(ix.gardenStreet(O, [W]).rows, [{ key: W, on: 1, state: 2 }], 'the first block of each seq stands') + t.alike(seqs(ix.garden(O, NOW, open).helps), [W + ':3']) + t.alike(ix.garden(W, NOW).given, [3]) +}) + +test('garden: hostile records and arguments never throw', (t) => { + const ix = createEngagementIndex() + const bad = [ + { type: 'garden', on: 'yes', at: iso(NOW) }, + { type: 'garden', at: iso(NOW) }, + { type: 'garden-state', pots: 'x', n: 1, at: iso(NOW) }, + { type: 'garden-help', to: O.toUpperCase(), pot: 0, at: iso(NOW) }, + { type: 'garden-help', to: { toString: () => O }, pot: 0, at: iso(NOW) }, + { type: 'garden-help', to: O, pot: 6, at: iso(NOW) }, + { type: 'garden-help', to: O, pot: 1.5, at: iso(NOW) }, + { type: 'garden-help', to: O, pot: 0, at: 5 }, + { type: 'garden-help', to: O, pot: 0, at: 'not a date' }, + { type: 'garden-help', to: O, pot: 0, get at () { throw new Error('x') } } + ] + bad.forEach((r, i) => t.is(ix.add(W, i + 1, r), false)) + t.is(ix.stats().records, 0) + for (const now of [NaN, Infinity, '1', null]) t.is(ix.garden(O, now), null) + t.is(ix.garden('x', NOW), null) + t.is(ix.garden(O, NOW, null), null, 'a null options bag') + t.is(ix.gardenStreet(O, null), null, 'followed must be iterable') + t.is(ix.gardenStreet(O, [W], { cursor: 'x' }), null) + t.is(ix.gardenStreet('x', [W]), null) + t.alike(ix.gardenStreet(O, [5, null, 'x', W.toUpperCase()]), { rows: [], next: null }, 'junk keys are skipped') + const throwing = { [Symbol.iterator] () { throw new Error('x') } } + t.is(ix.gardenStreet(O, throwing), null) + t.ok(add(ix, W, 20, gardenRecord({ on: true, at: iso(NOW) }))) + t.alike(ix.gardenStreet(O, [W], { visible: () => { throw new Error('x') } }).rows, [], 'a throwing visible hides') +}) + +test('gardenStreet: pages of GARDEN_STREET_PAGE rows, sorted keys, cursor strictly after', (t) => { + const ix = createEngagementIndex() + const page = G.GARDEN_STREET_PAGE + const keys = [] + for (let i = 0; i < page + 5; i++) { + keys.push(k(i)) + t.ok(add(ix, k(i), 1, gardenRecord({ on: true, at: iso(NOW) }))) + } + const followed = new Set(keys.slice().reverse()) + const first = ix.gardenStreet(O, followed) + t.is(first.rows.length, page) + t.alike(first.rows.map((r) => r.key), keys.slice(0, page)) + t.is(first.next, keys[page - 1]) + t.alike(first.rows[0], { key: k(0), on: 1, state: null }) + const second = ix.gardenStreet(O, followed, { cursor: first.next }) + t.alike(second.rows.map((r) => r.key), keys.slice(page)) + t.is(second.next, null) + const exact = ix.gardenStreet(O, keys.slice(0, page)) + t.is(exact.rows.length, page) + t.is(exact.next, null, 'a full page with no keys left ends the walk') + t.alike(ix.gardenStreet(O, keys, { visible: (key) => key !== k(0) }).rows[0].key, k(1), 'an invisible key is skipped') +}) + +test('gardenStreet: paging across the key budget never dead ends', (t) => { + const ix = createEngagementIndex() + const total = 2 * GARDEN_STREET_SCAN + 500 + const keys = [] + for (let i = 0; i < total; i++) keys.push(k(i)) + const onAt = [10, GARDEN_STREET_SCAN + 500, 2 * GARDEN_STREET_SCAN + 400] + for (const i of onAt) t.ok(add(ix, k(i), 1, gardenRecord({ on: true, at: iso(NOW) }))) + t.ok(add(ix, k(20), 1, gardenRecord({ on: false, at: iso(NOW) }))) + const got = [] + const cursors = [] + let cursor = null + for (let call = 0; call < 10; call++) { + const res = ix.gardenStreet(O, keys, { cursor }) + got.push(...res.rows.map((r) => r.key)) + cursors.push(res.next) + if (res.next === null) break + t.ok(cursor === null || res.next > cursor, 'the cursor always advances') + cursor = res.next + } + t.alike(got, onAt.map(k), 'every on balcony is found, the off one never') + t.alike(cursors, [k(GARDEN_STREET_SCAN - 1), k(2 * GARDEN_STREET_SCAN - 1), null], 'each call spends the whole budget') + const empty = ix.gardenStreet(O, keys.slice(0, GARDEN_STREET_SCAN)) + t.alike(empty, { rows: [{ key: k(10), on: 1, state: null }], next: null }, 'exactly the budget with nothing left ends the walk') +}) + +// The street cursor is public: a key the follows index hides must never be +// `next`, or paging /rua would reveal that follow edge. +test('gardenStreet: a key visible refuses is never listed, never next and costs no budget', (t) => { + const ix = createEngagementIndex() + const total = GARDEN_STREET_SCAN + 100 + const keys = [] + for (let i = 0; i < total; i++) keys.push(k(i)) + const refused = new Set([1997, 1998, 1999, 2000, 2001, 2002, 2003].map(k)) + // A few on balconies, refused ones among them; the rest are closed, so + // the key budget, not the page, ends a call. + const onAt = [5, 1998, 1999, 2004, 2050] + for (const i of onAt) t.ok(add(ix, k(i), 1, gardenRecord({ on: true, at: iso(NOW) }))) + const calls = new Map() + const visible = (key) => { + calls.set(key, (calls.get(key) || 0) + 1) + return !refused.has(key) + } + const got = [] + const cursors = [] + let cursor = null + for (let call = 0; call < 10; call++) { + const res = ix.gardenStreet(O, keys, { cursor, visible }) + got.push(...res.rows.map((r) => r.key)) + cursors.push(res.next) + if (res.next === null) break + t.absent(refused.has(res.next), 'next is never a refused key') + cursor = res.next + } + t.is(cursors.at(-1), null, 'the walk terminates') + t.alike(cursors, [k(GARDEN_STREET_SCAN + 6), null], 'only visible keys count toward the budget') + t.alike(got, [5, 2004, 2050].map(k), 'every visible on balcony once, the refused ones never') + t.ok(keys.every((key) => calls.get(key) === 1), 'every key examined exactly once across the walk') + + const allOn = createEngagementIndex() + for (const key of keys) t.ok(add(allOn, key, 1, gardenRecord({ on: true, at: iso(NOW) }))) + const seen = [] + cursor = null + for (let call = 0; call < 200; call++) { + const res = allOn.gardenStreet(O, keys, { cursor, visible: (key) => !refused.has(key) }) + seen.push(...res.rows.map((r) => r.key)) + if (res.next === null) break + t.absent(refused.has(res.next), 'next is never a refused key') + cursor = res.next + } + t.alike(seen, keys.filter((key) => !refused.has(key)), 'opted in everywhere: every visible key listed once, in order') +}) + +test('gardenStreet: a tail of refused keys after the last visible one ends the walk', (t) => { + const ix = createEngagementIndex() + const keys = [] + for (let i = 0; i < GARDEN_STREET_SCAN + 50; i++) keys.push(k(i)) + const visible = (key) => key <= k(GARDEN_STREET_SCAN - 1) + const first = ix.gardenStreet(O, keys, { visible }) + t.is(first.next, k(GARDEN_STREET_SCAN - 1), 'the budget ends on the last visible key') + t.alike(ix.gardenStreet(O, keys, { visible, cursor: first.next }), { rows: [], next: null }, 'only refused keys remain') + t.alike(ix.gardenStreet(O, [k(1), k(2)], { visible: (key) => key === k(1) }), { rows: [], next: null }) +}) + +test('gardenStreet: visible is asked for a closed or off balcony too', (t) => { + const ix = createEngagementIndex() + t.ok(add(ix, k(2), 1, gardenRecord({ on: false, at: iso(NOW) }))) + let calls = 0 + const res = ix.gardenStreet(O, [k(1), k(2)], { visible: () => { calls++; return true } }) + t.alike(res, { rows: [], next: null }) + t.is(calls, 2, 'one call per key, closed or off') +}) diff --git a/packages/core/test/profile-extra-index.test.js b/packages/core/test/profile-extra-index.test.js index 4e2d56de..c43900e5 100644 --- a/packages/core/test/profile-extra-index.test.js +++ b/packages/core/test/profile-extra-index.test.js @@ -6,7 +6,7 @@ // answer is an open object, so keys are asserted by name only. No network. const test = require('brittle') const { createEngagementIndex } = require('../lib/engagement-index') -const { profileAboutRecord, moodRecord, profileRecord, topFriendsRecord, visitsRecord, validRecordCopy } = require('../lib/records') +const { profileAboutRecord, moodRecord, profileRecord, topFriendsRecord, visitsRecord, gardenRecord, gardenStateRecord, validRecordCopy } = require('../lib/records') const A = 'a'.repeat(64) const B = 'b'.repeat(64) @@ -116,6 +116,8 @@ test('profileExtra: the answer carries the about, mood and top keys by name (lat t.is(got.top, null) t.ok('visits' in got) t.is(got.visits, null) + t.ok('garden' in got) + t.is(got.garden, null) }) test('profileExtra: visits is the newest live visits block (on or off), deletes fall back, slots and purge', (t) => { @@ -195,3 +197,36 @@ test('profileExtra: a record refused at the per-author cap leaves no pointer', ( t.is(ix.profileExtra(A).about, 1) t.is(ix.profileExtra(A).mood, null) }) + +test('profileExtra: garden is the newest live garden block (on or off), deletes fall back, slots and purge', (t) => { + const ix = createEngagementIndex() + t.is(ix.profileExtra(A).garden, null) + t.ok(add(ix, A, 1, gardenRecord({ on: true, at: at(1) }))) + t.is(ix.profileExtra(A).garden, 1) + t.ok(add(ix, A, 2, gardenRecord({ on: false, at: at(2) }))) + t.is(ix.profileExtra(A).garden, 2, 'an off block is still the pointer (the node peeks it)') + t.ok(add(ix, A, 3, gardenStateRecord({ pots: [null, null, null, null, null, null], n: 1, at: at(3) }))) + t.is(ix.profileExtra(A).garden, 2, 'a garden-state block is not the flag pointer') + t.ok(add(ix, A, 4, gardenRecord({ on: true, at: at(4) }))) + t.ok(add(ix, A, 4, gardenRecord({ on: true, at: at(4) })), 'a re-add is a no-op') + t.is(ix.profileExtra(A).garden, 4) + t.is(ix.stats().records, 4, 'each block holds one slot, once') + t.is(ix.profileExtra(A).visits, null, 'the other pointers are unaffected') + t.is(ix.profileExtra(B).garden, null, 'another author is unaffected') + t.ok(ix.removeRecord(A, 4)) + t.is(ix.profileExtra(A).garden, 2, 'a deleted block falls back, even to an off one') + t.ok(add(ix, A, 4, gardenRecord({ on: true, at: at(4) })), 'a tombstoned seq is not resurrected') + t.is(ix.profileExtra(A).garden, 2) + t.ok(ix.removeRecord(A, 2)) + t.ok(ix.removeRecord(A, 1)) + t.is(ix.profileExtra(A).garden, null) + t.ok(ix.removeRecord(A, 3)) + t.is(ix.stats().records, 0, 'every slot was released') + add(ix, B, 1, gardenRecord({ on: true, at: at(5) })) + ix.removeDiary(B) + t.is(ix.profileExtra(B).garden, null, 'removeDiary purges it') + t.is(ix.stats().records, 0) + add(ix, A, 7, gardenRecord({ on: true, at: at(6) })) + ix.clear() + t.is(ix.profileExtra(A).garden, null, 'clear purges it') +})