diff --git a/README.md b/README.md index dec30b78..ff826992 100644 --- a/README.md +++ b/README.md @@ -660,6 +660,14 @@ How many blocks are contiguously available starting from the first block of this Populated after `ready` has been emitted. Will be `0` before the event. +#### `core.tailContiguousLength` + +How many blocks are contiguously available locally, counting backward from the shared default core's current end until the first missing block. Will be `0` if the core is empty or its final block is missing. + +Like `core.contiguousLength`, this describes the shared default core, not the calling session's own end. Named sessions and snapshots can have a different `core.length` while still returning this shared count. + +Populated after `ready` has been emitted. Will be `0` before the event. + #### `core.remoteContiguousLength` How many blocks are contiguously available starting from the first block of this core on any known remote. This is only updated when a remote thinks it is fully contiguous such that they have all known blocks. diff --git a/index.js b/index.js index 8a08a290..7dc507fc 100644 --- a/index.js +++ b/index.js @@ -677,6 +677,11 @@ class Hypercore extends EventEmitter { return Math.min(this.core.state.length, this.core.header.hints.contiguousLength) } + get tailContiguousLength() { + if (this.opened === false) return 0 + return this.core.tailContiguousLength + } + get contiguousByteLength() { return 0 } diff --git a/lib/audit.js b/lib/audit.js index cb8cd8f6..6620e5e6 100644 --- a/lib/audit.js +++ b/lib/audit.js @@ -121,6 +121,7 @@ module.exports = async function auditCore( if (!tx && !stats.corrupt) tx = core.state.storage.write() core.bitfield.set(index, false) + core._tailContiguousStart = -1 // direct write bypasses Core._setBitfieldRanges const page = core.bitfield.getBitfield(index) if (page.bitfield) tx.putBitfieldPage(page.index, page.bitfield) diff --git a/lib/copy-prologue.js b/lib/copy-prologue.js index 51c845b4..0337ed0c 100644 --- a/lib/copy-prologue.js +++ b/lib/copy-prologue.js @@ -150,6 +150,7 @@ async function flushBatch(prologue, src, dst, batch) { const roots = nodes.slice(0, batch.roots.length) dst.state.setRoots(roots) dst.header.tree = prologueToTree(prologue) + dst._tailContiguousStart = -1 // head upgraded outside Core._setBitfieldRanges } if (userData.length > 0) { @@ -176,6 +177,7 @@ async function flushBatch(prologue, src, dst, batch) { start = index dst.bitfield.set(index, true) + dst._tailContiguousStart = -1 // direct write bypasses Core._setBitfieldRanges } if (length > 0) signalReplicator(dst, upgraded, start, length) diff --git a/lib/core.js b/lib/core.js index 6c1b5353..40669b84 100644 --- a/lib/core.js +++ b/lib/core.js @@ -63,6 +63,8 @@ module.exports = class Core { this._verifiesFlushed = null this._legacy = !!opts.legacy this._repairMode = false + this._tailContiguousStart = -1 + this._tailContiguousEnd = 0 this.opening = this._open(opts) this.opening.catch(noop) @@ -869,6 +871,46 @@ module.exports = class Core { _setBitfieldRanges(start, end, value) { this.bitfield.setRange(start, end, value) if (this.skipBitfield !== null) this.skipBitfield.setRange(start, end, value) + + if (start >= end || this._tailContiguousStart === -1) return + + if (value) { + if (start <= this._tailContiguousEnd && end >= this._tailContiguousStart) { + if (end > this._tailContiguousEnd) this._tailContiguousEnd = end + if (start < this._tailContiguousStart) { + this._tailContiguousStart = this.bitfield.lastUnset(start - 1) + 1 + } + } + } else if (end > this._tailContiguousStart && start < this._tailContiguousEnd) { + this._tailContiguousStart = Math.min(end, this._tailContiguousEnd) + } + } + + get tailContiguousLength() { + const length = this.state.length + + if (this._tailContiguousStart === -1 || length < this._tailContiguousEnd) { + const start = length === 0 ? 0 : this.bitfield.lastUnset(length - 1) + 1 + this._tailContiguousStart = start + this._tailContiguousEnd = length + return length - start + } + + if (length === this._tailContiguousEnd) return length - this._tailContiguousStart + + // Retain the known tail when the new head is missing. Downloading just that head + // can then extend the cached interval without rescanning the old suffix. + if (!this.bitfield.get(length - 1)) return 0 + + if (this.bitfield.firstUnset(this._tailContiguousEnd) >= length) { + this._tailContiguousEnd = length + return length - this._tailContiguousStart + } + + const start = this.bitfield.lastUnset(length - 1) + 1 + this._tailContiguousStart = start + this._tailContiguousEnd = length + return length - start } close() { diff --git a/test/all.js b/test/all.js index c79c360a..aa839ac4 100644 --- a/test/all.js +++ b/test/all.js @@ -38,6 +38,7 @@ async function runTests() { await import('./sessions.js') await import('./snapshots.js') await import('./streams.js') + await import('./tail-contiguous-length.js') await import('./timeouts.js') await import('./user-data.js') await import('./wants.js') diff --git a/test/basic.js b/test/basic.js index d55ada39..7f7d7a23 100644 --- a/test/basic.js +++ b/test/basic.js @@ -869,6 +869,37 @@ test('contiguousLength gets updated after an append (also on disk)', async funct } }) +test('tail contiguous length follows holes cleared from the end', async function (t) { + const core = await create(t) + + t.is(core.tailContiguousLength, 0, 'empty core has no tail blocks') + + await core.append(['a', 'b', 'c', 'd', 'e']) + t.is(core.tailContiguousLength, 5, 'every appended block is present') + + await core.clear(0, 2) + t.is(core.length, 5, 'clearing a prefix keeps the logical length') + t.is(core.contiguousLength, 0, 'forward contiguous length begins at the first hole') + t.is( + core.tailContiguousLength, + 3, + 'tail counts present blocks from the end, not the last hole index' + ) + + await core.clear(3) + t.is(core.tailContiguousLength, 1, 'a hole at 3 leaves only the last block contiguous') + + await core.clear(4) + t.is( + core.tailContiguousLength, + 0, + 'a missing last block gives zero even when an earlier block is present' + ) + + await core.append(['f']) + t.is(core.tailContiguousLength, 1, 'a new block after a tail hole restarts the run') +}) + test('append alignment to bitfield boundary', async function (t) { const tmpDir = await t.tmp() @@ -891,6 +922,7 @@ test('append alignment to bitfield boundary', async function (t) { t.alike(getBitfields(core, 0, 32769), expectedBitfields) t.is(core.contiguousLength, 32768) t.is(core.core.header.hints.contiguousLength, 32768) + t.is(core.tailContiguousLength, 32768, 'the tail run fills the first bitfield page exactly') await core.close() } @@ -903,6 +935,11 @@ test('append alignment to bitfield boundary', async function (t) { t.alike(getBitfields(core, 0, 32769), expectedBitfields) t.is(core.contiguousLength, 32768) t.is(core.core.header.hints.contiguousLength, 32768) + t.is( + core.tailContiguousLength, + 32768, + 'the tail recalculates to the first bitfield page boundary on reopen' + ) await core.close() } diff --git a/test/core.js b/test/core.js index d8d906c1..dfc300b2 100644 --- a/test/core.js +++ b/test/core.js @@ -243,6 +243,44 @@ test('core - verify parallel upgrades', async function (t) { t.alike(tree2.signature, tree1.signature) }) +test('core - audit bitfield repair invalidates the tail cache', async function (t) { + const { core } = await create(t) + + await core.state.append([b4a.from('0'), b4a.from('1'), b4a.from('2'), b4a.from('3')]) + t.is(core.tailContiguousLength, 4) + + const tx = core.state.storage.write() + tx.deleteBlock(3) + await tx.flush() + + const stats = await core.audit({ tree: false, blocks: false, bitfield: true }) + + t.is(stats.droppedBits, 1) + t.is(core.tailContiguousLength, 0) +}) + +test('core - prologue copy invalidates the tail cache', async function (t) { + const { core: src } = await create(t) + + await src.state.append([b4a.from('a'), b4a.from('b')]) + + const manifest = { ...src.header.manifest } + manifest.prologue = { length: src.state.length, hash: await src.state.hash() } + + const { core: dst } = await create(t, { manifest }) + + const batch = src.state.createTreeBatch() + const proof = await getProof(src, { upgrade: { start: 0, length: src.state.length } }) + proof.upgrade.signature = dst.verifier.sign(batch, src.header.keyPair) + + t.ok(await dst.verify(proof)) + t.is(dst.state.length, 2) + t.is(dst.tailContiguousLength, 0, 'the upgraded head has no blocks yet') + + await dst.copyPrologue(src.state) + t.is(dst.tailContiguousLength, 2, 'copied prologue bits invalidate the empty tail') +}) + test('core - clone', async function (t) { const { core } = await create(t) diff --git a/test/tail-contiguous-length.js b/test/tail-contiguous-length.js new file mode 100644 index 00000000..ce4d57d7 --- /dev/null +++ b/test/tail-contiguous-length.js @@ -0,0 +1,330 @@ +const test = require('brittle') +const { create, createStored, replicate, unreplicate, eventFlush } = require('./helpers') + +function trackScans(core) { + const bitfield = core.core.bitfield + const scans = [] + + for (const method of ['lastUnset', 'firstUnset']) { + const original = bitfield[method].bind(bitfield) + bitfield[method] = function (position) { + scans.push({ method, position }) + return original(position) + } + } + + return scans +} + +test('tailContiguousLength - repeated reads and sessions reuse the first scan', async function (t) { + const core = await create(t) + + await core.append(['a', 'b', 'c', 'd', 'e']) + + const scans = trackScans(core) + + t.is(core.tailContiguousLength, 5) + t.alike(scans, [{ method: 'lastUnset', position: 4 }]) + + t.is(core.tailContiguousLength, 5) + t.is(core.tailContiguousLength, 5) + t.is(scans.length, 1, 'repeated reads do not rescan the tail') + + const named = core.session({ name: 'reader' }) + await named.ready() + + const snap = core.snapshot() + await snap.ready() + + t.is(named.tailContiguousLength, 5) + t.is(snap.tailContiguousLength, 5) + t.is(scans.length, 1, 'sessions share the core cache') + + await named.close() + await snap.close() +}) + +test('tailContiguousLength - downloads only scan before the written tail', async function (t) { + const a = await create(t) + await a.append(['0', '1', '2', '3', '4', '5', '6', '7']) + + const b = await create(t, a.key) + replicate(a, b, t) + + await b.download({ start: 5, end: 8 }).done() + + const scans = trackScans(b) + t.is(b.tailContiguousLength, 3) + t.alike(scans, [{ method: 'lastUnset', position: 7 }]) + + t.is(b.tailContiguousLength, 3) + t.is(scans.length, 1, 'repeated reads reuse the tail scan') + + const downloads = [] + b.on('download', (index) => downloads.push([index, b.tailContiguousLength])) + + await b.get(1) + await b.get(2) + t.alike( + downloads, + [ + [1, 3], + [2, 3] + ], + 'real earlier blocks downloaded' + ) + t.is(b.tailContiguousLength, 3, 'an unrelated earlier download leaves the tail cached') + t.is(scans.length, 1, 'unrelated earlier blocks do not scan the known tail') + + downloads.length = 0 + await b.get(3) + await b.get(4) + t.alike( + downloads, + [ + [3, 3], + [4, 7] + ], + 'real boundary blocks downloaded' + ) + t.is(b.tailContiguousLength, 7, 'the tail now reaches block 1, block 0 is still missing') + t.alike( + scans.slice(1), + [{ method: 'lastUnset', position: 3 }], + 'only the join at the cached boundary scans' + ) +}) + +test('tailContiguousLength - a missing head retains the known tail across growth', async function (t) { + const a = await create(t) + await a.append(['0', '1', '2', '3', '4']) + + const b = await create(t, a.key) + replicate(a, b, t) + await b.download({ start: 0, end: 5 }).done() + + t.is(b.tailContiguousLength, 5) + + const scans = trackScans(b) + + await a.append('5') + await b.update({ wait: true }) + + t.is(b.length, 6) + t.is(b.tailContiguousLength, 0) + t.is(b.tailContiguousLength, 0) + t.is(scans.length, 0, 'a missing newest block does not rescan the retained tail') + + const downloads = [] + b.on('download', (index) => downloads.push([index, b.tailContiguousLength])) + await b.get(5) + + t.alike(downloads, [[5, 6]]) + t.is(b.tailContiguousLength, 6, 'the downloaded head extends the retained run') + t.is(scans.length, 0, 'the retained tail suffix is not rescanned') +}) + +test('tailContiguousLength - descending head downloads close the gap once', async function (t) { + const a = await create(t) + await a.append(['0', '1', '2', '3', '4']) + + const b = await create(t, a.key) + replicate(a, b, t) + await b.download({ start: 0, end: 5 }).done() + + t.is(b.tailContiguousLength, 5) + + const scans = trackScans(b) + + await a.append('5') + await b.update({ wait: true }) + await a.append('6') + await b.update({ wait: true }) + await a.append('7') + await b.update({ wait: true }) + + t.is(b.length, 8) + t.is(b.tailContiguousLength, 0) + t.is(b.tailContiguousLength, 0) + t.is(scans.length, 0, 'repeated head-only growth keeps the old tail cached') + + const downloads = [] + b.on('download', (index) => downloads.push([index, b.tailContiguousLength])) + + await b.get(7) + t.is(b.tailContiguousLength, 1) + + await b.get(6) + t.is(b.tailContiguousLength, 2) + + await b.get(5) + t.is(b.tailContiguousLength, 8) + + t.alike(downloads, [ + [7, 1], + [6, 2], + [5, 8] + ]) + t.alike(scans, [ + { method: 'firstUnset', position: 5 }, + { method: 'lastUnset', position: 7 }, + { method: 'lastUnset', position: 5 }, + { method: 'lastUnset', position: 4 } + ]) + + t.is(b.tailContiguousLength, 8) + t.is(scans.length, 4, 'repeated reads stay cached after the gap closes') +}) + +test('tailContiguousLength - in-place writes update the cached tail', async function (t) { + const core = await create(t) + await core.append(['a', 'b', 'c', 'd', 'e']) + + t.is(core.tailContiguousLength, 5) + + const scans = trackScans(core) + const appendTails = [] + core.once('append', () => appendTails.push(core.tailContiguousLength)) + + await core.append(['f', 'g']) + t.alike(appendTails, [7], 'the append event sees the extended tail') + t.is(core.tailContiguousLength, 7) + t.is(scans.length, 0, 'an ordinary append extends the cache without scanning') + + await core.clear(0, 2) + t.is(core.tailContiguousLength, 5, 'clearing a prefix does not affect the tail') + t.is(scans.length, 0) + + await core.clear(5, 6) + t.is(core.tailContiguousLength, 1, 'clearing inside the tail keeps the right remainder') + t.is(scans.length, 0) + + await core.clear(6) + t.is(core.tailContiguousLength, 0, 'clearing the last block empties the tail') + t.is(scans.length, 0) + + await core.append(['h']) + t.is(core.tailContiguousLength, 1, 'appending after a tail hole restarts the run') + t.is(scans.length, 0) +}) + +test('tailContiguousLength - truncation then sparse regrowth without a read', async function (t) { + const a = await create(t) + await a.append(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']) + + const b = await create(t, { keyPair: a.keyPair }) + const streams = replicate(a, b, t) + await b.download({ start: 0, end: 10 }).done() + + t.is(b.tailContiguousLength, 10) + + await unreplicate(streams) + + await b.truncate(5) + await a.truncate(5) + await a.append(['5', '6', '7', '8', '9']) + + t.is(a.length, 10) + t.is(b.length, 5) + + const appended = new Promise((resolve) => b.once('append', resolve)) + replicate(a, b, t) + await appended + await eventFlush() + + t.is(b.length, 10) + t.is(b.tailContiguousLength, 0, 'a sparse regrowth does not resurrect truncated blocks') + + await b.download({ start: 5, end: 10 }).done() + t.is(b.tailContiguousLength, 10, 'downloaded regrown blocks restore the tail') +}) + +test('tailContiguousLength - equal length reorg clears the replaced tail', async function (t) { + const a = await create(t) + await a.append(['a', 'b', 'c', 'd', 'e', 'f']) + + const b = await create(t, a.key) + const streams = replicate(a, b, t) + await b.download({ start: 0, end: 6 }).done() + + t.is(b.tailContiguousLength, 6) + + await unreplicate(streams) + + await a.truncate(3) + await a.append(['d2', 'e2', 'f2']) + + t.is(a.length, 6) + t.is(a.fork, 1) + + const truncated = new Promise((resolve) => b.once('truncate', resolve)) + replicate(a, b, t) + await truncated + await eventFlush() + + t.is(b.length, 6) + t.is(b.tailContiguousLength, 0, 'replaced blocks are gone even though the length is unchanged') + + await b.download({ start: 3, end: 6 }).done() + t.is(b.tailContiguousLength, 6) +}) + +test('tailContiguousLength - atomic commits keep the shared default semantics', async function (t) { + const core = await create(t) + await core.append(['a', 'b', 'c', 'd', 'e']) + + t.is(core.tailContiguousLength, 5) + + const atom = core.state.storage.createAtom() + const atomic = core.session({ atom }) + + await atomic.truncate(3) + t.is(atomic.tailContiguousLength, 5, 'an atomic draft keeps the shared default tail') + t.is(core.tailContiguousLength, 5) + + const truncateTails = [] + core.once('truncate', () => truncateTails.push(core.tailContiguousLength)) + + await atom.flush() + t.alike(truncateTails, [3], 'the truncate event sees the committed tail') + t.is(core.tailContiguousLength, 3) + + await atomic.close() + + const atom2 = core.state.storage.createAtom() + const atomic2 = core.session({ atom: atom2 }) + + await atomic2.append('f') + t.is(core.tailContiguousLength, 3, 'a draft append is not visible on the default core') + + const appendTails = [] + core.once('append', () => appendTails.push(core.tailContiguousLength)) + + await atom2.flush() + t.alike(appendTails, [4], 'the append event sees the committed tail') + t.is(core.tailContiguousLength, 4) + + await atomic2.close() +}) + +test('tailContiguousLength - a reopened sparse core scans once then reuses', async function (t) { + const createCore = await createStored(t) + + const a = await createCore() + await a.ready() + await a.append(['0', '1', '2', '3', '4', '5']) + await a.clear(1) + await a.close() + + const b = await createCore() + await b.ready() + t.teardown(() => b.close()) + + const scans = trackScans(b) + + t.is(b.tailContiguousLength, 4) + t.alike(scans, [{ method: 'lastUnset', position: 5 }]) + + t.is(b.tailContiguousLength, 4) + t.is(scans.length, 1, 'a reopened sparse core reuses its first scan') +})