Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -660,6 +660,14 @@ How many blocks are contiguously available starting from the first block of this

Populated after `ready` has been emitted. Will be `0` before the event.

#### `core.tailContiguousLength`

How many blocks are contiguously available locally, counting backward from the shared default core's current end until the first missing block. Will be `0` if the core is empty or its final block is missing.

Like `core.contiguousLength`, this describes the shared default core, not the calling session's own end. Named sessions and snapshots can have a different `core.length` while still returning this shared count.

Populated after `ready` has been emitted. Will be `0` before the event.

#### `core.remoteContiguousLength`

How many blocks are contiguously available starting from the first block of this core on any known remote. This is only updated when a remote thinks it is fully contiguous such that they have all known blocks.
Expand Down
5 changes: 5 additions & 0 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -677,6 +677,11 @@ class Hypercore extends EventEmitter {
return Math.min(this.core.state.length, this.core.header.hints.contiguousLength)
}

get tailContiguousLength() {
if (this.opened === false) return 0
return this.core.tailContiguousLength
}
Comment thread
mynameisdaniil marked this conversation as resolved.
Resolved

get contiguousByteLength() {
return 0
}
Expand Down
1 change: 1 addition & 0 deletions lib/audit.js
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ module.exports = async function auditCore(
if (!tx && !stats.corrupt) tx = core.state.storage.write()

core.bitfield.set(index, false)
core._tailContiguousStart = -1 // direct write bypasses Core._setBitfieldRanges

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should consider changing this bitfield set call to the core._setBitfieldRange() equivalent, but that should be in a separate PR since it also modies the skip bitfield and is outside the scope here.


const page = core.bitfield.getBitfield(index)
if (page.bitfield) tx.putBitfieldPage(page.index, page.bitfield)
Expand Down
2 changes: 2 additions & 0 deletions lib/copy-prologue.js
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ async function flushBatch(prologue, src, dst, batch) {
const roots = nodes.slice(0, batch.roots.length)
dst.state.setRoots(roots)
dst.header.tree = prologueToTree(prologue)
dst._tailContiguousStart = -1 // head upgraded outside Core._setBitfieldRanges

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This shouldn't be necessary right?

  1. Its not updating the bitfield here.
  2. This only runs if there is no head entry so there should be no blocks. IE this should always be -1 already.

}

if (userData.length > 0) {
Expand All @@ -176,6 +177,7 @@ async function flushBatch(prologue, src, dst, batch) {

start = index
dst.bitfield.set(index, true)
dst._tailContiguousStart = -1 // direct write bypasses Core._setBitfieldRanges

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as audit, we should consider changing the bitfield set to dist._setBitfieldRange() equivalent, but not now.

}

if (length > 0) signalReplicator(dst, upgraded, start, length)
Expand Down
42 changes: 42 additions & 0 deletions lib/core.js
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ module.exports = class Core {
this._verifiesFlushed = null
this._legacy = !!opts.legacy
this._repairMode = false
this._tailContiguousStart = -1
this._tailContiguousEnd = 0

this.opening = this._open(opts)
this.opening.catch(noop)
Expand Down Expand Up @@ -869,6 +871,46 @@ module.exports = class Core {
_setBitfieldRanges(start, end, value) {
this.bitfield.setRange(start, end, value)
if (this.skipBitfield !== null) this.skipBitfield.setRange(start, end, value)

if (start >= end || this._tailContiguousStart === -1) return

if (value) {
if (start <= this._tailContiguousEnd && end >= this._tailContiguousStart) {
if (end > this._tailContiguousEnd) this._tailContiguousEnd = end
if (start < this._tailContiguousStart) {
this._tailContiguousStart = this.bitfield.lastUnset(start - 1) + 1
}
}
} else if (end > this._tailContiguousStart && start < this._tailContiguousEnd) {
this._tailContiguousStart = Math.min(end, this._tailContiguousEnd)
}
}

get tailContiguousLength() {
const length = this.state.length

if (this._tailContiguousStart === -1 || length < this._tailContiguousEnd) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The _tailContiguousEnd should never be greater than the length. That would mean we didn't update the property correctly elsewhere.

Looking into this, in _setBitfieldRanges a case is missing where the updated range exceeds the current length. This should crop the tail contiguous end to the length or invalidate the cached values entirely if the length was truncated to before the previous length:

    } else if (end > this.state.length && this._tailContiguousEnd > this.state.length) {
      const length = this.state.length

      if (this._tailContiguousStart >= length) this._tailContiguousStart = -1
      else this._tailContiguousEnd = length

I placed this before the last check that sets the cached start.

const start = length === 0 ? 0 : this.bitfield.lastUnset(length - 1) + 1

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
const start = length === 0 ? 0 : this.bitfield.lastUnset(length - 1) + 1
const start = this.bitfield.lastUnset(length - 1) + 1

This is already guarded in the findLast() of the bitfield for the 0 case. It returns -1 which when + 1 is applied becomes the 0 the ternary used.

this._tailContiguousStart = start
this._tailContiguousEnd = length
return length - start
}

if (length === this._tailContiguousEnd) return length - this._tailContiguousStart

// Retain the known tail when the new head is missing. Downloading just that head
// can then extend the cached interval without rescanning the old suffix.
if (!this.bitfield.get(length - 1)) return 0

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Leaving some thoughts on how this will be used for reference if it comes up later.

While this is correct, this also means that each append event will cause the tailContiguousLength to be 0 for the event briefly since the upgrade comes through first and then the block is downloaded.

This probably means that append is just not the right event to detect additive tailContiguousLength changes. download would be more appropriate in those cases. But it does indicate that maybe a tail-contiguous-length event might be useful. We'll see when integrating elsewhere.


if (this.bitfield.firstUnset(this._tailContiguousEnd) >= length) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Initially this seemed to indicate we were missing an update elsewhere to the cached range, but walking through it I realized that the reason this is here is for situations where the tail is filled in before extending the previous range through. In that scenario the cached range could be updated by scanning the bitfield, but this would likely be a more expensive scan.

After that rabbit trail, I realized that the .firstUnset() scan is unneccessary and can fall through to the full scan below. This is because:

  1. it still scans and the case that it returns early is longer than the scan below.
  2. It only updates the ._tailContiguousEnd to the length anyways. Same as the full scan.
    So I think it can be skipped as doing a full scan only uses one scan and also ensures the start is accurate.

Given that, the full scan also happens twice, once for the -1 start case and again as the last case. So we can flip the early execution around so it checks non-full scale scenarios first and falls through to the full scan once. It would look like this:

  get tailContiguousLength() {
    const length = this.state.length

    if (this._tailContiguousStart !== -1) {
      if (length === this._tailContiguousEnd) return length - this._tailContiguousStart

      // Retain the known tail when the new head is missing. Downloading just that head
      // can then extend the cached interval without rescanning the old suffix.
      if (!this.bitfield.get(length - 1)) return 0
    }

    const start = this.bitfield.lastUnset(length - 1) + 1
    this._tailContiguousStart = start
    this._tailContiguousEnd = length
    return length - start
  }

This has the nice side effect that all early returns dont update the cache and the fallback is the only place that does the most expensive scan and updates the cached values.

this._tailContiguousEnd = length
return length - this._tailContiguousStart
}

const start = this.bitfield.lastUnset(length - 1) + 1
this._tailContiguousStart = start
this._tailContiguousEnd = length
return length - start
}

close() {
Expand Down
1 change: 1 addition & 0 deletions test/all.js
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ async function runTests() {
await import('./sessions.js')
await import('./snapshots.js')
await import('./streams.js')
await import('./tail-contiguous-length.js')
await import('./timeouts.js')
await import('./user-data.js')
await import('./wants.js')
Expand Down
37 changes: 37 additions & 0 deletions test/basic.js
Original file line number Diff line number Diff line change
Expand Up @@ -869,6 +869,37 @@ test('contiguousLength gets updated after an append (also on disk)', async funct
}
})

test('tail contiguous length follows holes cleared from the end', async function (t) {
const core = await create(t)

t.is(core.tailContiguousLength, 0, 'empty core has no tail blocks')

await core.append(['a', 'b', 'c', 'd', 'e'])
t.is(core.tailContiguousLength, 5, 'every appended block is present')

await core.clear(0, 2)
t.is(core.length, 5, 'clearing a prefix keeps the logical length')
t.is(core.contiguousLength, 0, 'forward contiguous length begins at the first hole')
t.is(
core.tailContiguousLength,
3,
'tail counts present blocks from the end, not the last hole index'
)

await core.clear(3)
t.is(core.tailContiguousLength, 1, 'a hole at 3 leaves only the last block contiguous')

await core.clear(4)
t.is(
core.tailContiguousLength,
0,
'a missing last block gives zero even when an earlier block is present'
)

await core.append(['f'])
t.is(core.tailContiguousLength, 1, 'a new block after a tail hole restarts the run')
})

test('append alignment to bitfield boundary', async function (t) {
const tmpDir = await t.tmp()

Expand All @@ -891,6 +922,7 @@ test('append alignment to bitfield boundary', async function (t) {
t.alike(getBitfields(core, 0, 32769), expectedBitfields)
t.is(core.contiguousLength, 32768)
t.is(core.core.header.hints.contiguousLength, 32768)
t.is(core.tailContiguousLength, 32768, 'the tail run fills the first bitfield page exactly')

await core.close()
}
Expand All @@ -903,6 +935,11 @@ test('append alignment to bitfield boundary', async function (t) {
t.alike(getBitfields(core, 0, 32769), expectedBitfields)
t.is(core.contiguousLength, 32768)
t.is(core.core.header.hints.contiguousLength, 32768)
t.is(
core.tailContiguousLength,
32768,
'the tail recalculates to the first bitfield page boundary on reopen'
)

await core.close()
}
Expand Down
38 changes: 38 additions & 0 deletions test/core.js
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,44 @@ test('core - verify parallel upgrades', async function (t) {
t.alike(tree2.signature, tree1.signature)
})

test('core - audit bitfield repair invalidates the tail cache', async function (t) {
const { core } = await create(t)

await core.state.append([b4a.from('0'), b4a.from('1'), b4a.from('2'), b4a.from('3')])
t.is(core.tailContiguousLength, 4)

const tx = core.state.storage.write()
tx.deleteBlock(3)
await tx.flush()

const stats = await core.audit({ tree: false, blocks: false, bitfield: true })

t.is(stats.droppedBits, 1)
t.is(core.tailContiguousLength, 0)
})

test('core - prologue copy invalidates the tail cache', async function (t) {
const { core: src } = await create(t)

await src.state.append([b4a.from('a'), b4a.from('b')])

const manifest = { ...src.header.manifest }
manifest.prologue = { length: src.state.length, hash: await src.state.hash() }

const { core: dst } = await create(t, { manifest })

const batch = src.state.createTreeBatch()
const proof = await getProof(src, { upgrade: { start: 0, length: src.state.length } })
proof.upgrade.signature = dst.verifier.sign(batch, src.header.keyPair)

t.ok(await dst.verify(proof))
t.is(dst.state.length, 2)
t.is(dst.tailContiguousLength, 0, 'the upgraded head has no blocks yet')

await dst.copyPrologue(src.state)
t.is(dst.tailContiguousLength, 2, 'copied prologue bits invalidate the empty tail')
})

test('core - clone', async function (t) {
const { core } = await create(t)

Expand Down
Loading
Loading