From 84874ee577aedf88581d859523ca933d03f35f84 Mon Sep 17 00:00:00 2001 From: generaltony007 <191973953+hakarrr@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:26:52 +0530 Subject: [PATCH 1/8] Remove timed-out block requests from the replication queue BlockRequest._unref() set queued = false but left the request in replicator._queued, which is only drained while a peer is connected. On a core with no peers, every block request that timed out or was cancelled stayed in the queue until the core closed, leaking one BlockRequest per timed-out get. Co-authored-by: generaltony007 <144252170+GeneralTony007@users.noreply.github.com> --- lib/replicator.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/replicator.js b/lib/replicator.js index 24bb15d9..536b97f2 100644 --- a/lib/replicator.js +++ b/lib/replicator.js @@ -214,6 +214,12 @@ class BlockRequest extends Attachable { if (this.resolved) return this.queued = false + // Also leave the replication queue: nothing else removes the request + // while no peer is connected, so it would stay there until the core + // closes. + const index = this.replicator._queued.indexOf(this) + if (index > -1) this.replicator._queued.splice(index, 1) + for (const req of this.inflight) { req.peer._cancelRequest(req) } From 8c9587d75a719ddfdf54a56255555e0d02312a27 Mon Sep 17 00:00:00 2001 From: generaltony007 <191973953+hakarrr@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:27:34 +0530 Subject: [PATCH 2/8] Test that timed-out block requests leave the replication queue On a core with no peers, run gets that time out and assert that the replication queue is empty afterwards. Without the fix, each timed-out request stays in replicator._queued until the core closes. Co-authored-by: generaltony007 <144252170+GeneralTony007@users.noreply.github.com> --- test/replicate.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/replicate.js b/test/replicate.js index c850b875..2d5a6d67 100644 --- a/test/replicate.js +++ b/test/replicate.js @@ -3521,4 +3521,16 @@ test('stream that closes while the core opens is not attached', async function ( await core.close() }) +test('timed-out block requests are removed from the replication queue', async function (t) { + const a = await create(t) + const b = await create(t, a.key) + + // b never connects to a peer, so its requests can only time out + for (let i = 0; i < 5; i++) { + await t.exception(b.get(i, { timeout: 50 })) + } + + t.is(b.core.replicator._queued.length, 0, 'timed-out requests leave the queue') +}) + function noop() {} From 071440c54446b2e045dd3fb0ce1939bc7aa17e46 Mon Sep 17 00:00:00 2001 From: generaltony007 <191973953+hakarrr@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:14:15 +0530 Subject: [PATCH 3/8] Compact the replication queue instead of splicing per cancellation Review feedback: indexOf + splice made each unreference O(n) in the queue length, so mass cancellation became quadratic. Stale entries are now counted and the queue is compacted in a single amortized pass from _queueBlock once they dominate, keeping each unreference O(1). A connected peer's sweep still drops stale entries as before. Co-authored-by: generaltony007 <144252170+GeneralTony007@users.noreply.github.com> --- lib/replicator.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/replicator.js b/lib/replicator.js index 536b97f2..335d8dcc 100644 --- a/lib/replicator.js +++ b/lib/replicator.js @@ -212,13 +212,14 @@ class BlockRequest extends Attachable { _unref() { if (this.resolved) return - this.queued = false // Also leave the replication queue: nothing else removes the request // while no peer is connected, so it would stay there until the core - // closes. - const index = this.replicator._queued.indexOf(this) - if (index > -1) this.replicator._queued.splice(index, 1) + // closes. Removal is amortized - _queueBlock compacts the queue in a + // single pass once stale entries dominate, so unreferencing many + // queued requests at once stays linear instead of quadratic. + if (this.queued === true) this.replicator._queuedStale++ + this.queued = false for (const req of this.inflight) { req.peer._cancelRequest(req) @@ -2050,6 +2051,7 @@ module.exports = class Replicator { this._alwaysLatestBlockEnabled = !!alwaysLatestBlock this._alwaysLatestBlock = this._alwaysLatestBlockEnabled ? 1 : 0 this._queued = [] + this._queuedStale = 0 this._seeks = [] this._upgrade = null @@ -2519,6 +2521,13 @@ module.exports = class Replicator { if (b.inflight.length > 0 || b.queued === true) return b.queued = true this._queued.push(b) + + // Stale entries are compacted here in one pass so that each + // unreference is O(1); a sweep with a connected peer drops them too. + if (this._queuedStale > 16 && this._queuedStale * 2 > this._queued.length) { + this._queued = this._queued.filter((b) => b.queued === true) + this._queuedStale = 0 + } } _resolveHashLocally(peer, req) { From 90ea99adb4dfac5c93bed98dcf39541e6875a7cd Mon Sep 17 00:00:00 2001 From: generaltony007 <191973953+hakarrr@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:15:03 +0530 Subject: [PATCH 4/8] Assert REQUEST_TIMEOUT and bounded queue retention in the timeout test Review feedback: the test accepted any error, so it could pass without exercising the timeout path. Now matches REQUEST_TIMEOUT, runs 100 timed-out requests, and asserts the queue retains at most the amortization bound (17) instead of all of them. Co-authored-by: generaltony007 <144252170+GeneralTony007@users.noreply.github.com> --- test/replicate.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/replicate.js b/test/replicate.js index 2d5a6d67..ac1cdb0a 100644 --- a/test/replicate.js +++ b/test/replicate.js @@ -3526,11 +3526,14 @@ test('timed-out block requests are removed from the replication queue', async fu const b = await create(t, a.key) // b never connects to a peer, so its requests can only time out - for (let i = 0; i < 5; i++) { - await t.exception(b.get(i, { timeout: 50 })) + for (let i = 0; i < 100; i++) { + await t.exception(b.get(i, { timeout: 20 }), /REQUEST_TIMEOUT/) } - t.is(b.core.replicator._queued.length, 0, 'timed-out requests leave the queue') + // Removal is amortized: _queueBlock compacts the queue once more than 16 + // stale entries dominate it, so the queue never retains more than 17 of + // the 100 timed-out requests (previously it retained all of them). + t.ok(b.core.replicator._queued.length <= 17, 'timed-out requests leave the queue') }) function noop() {} From 65fde7906ca1dc6826e1a89d17fdb033078244df Mon Sep 17 00:00:00 2001 From: generaltony007 <191973953+hakarrr@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:48:46 +0530 Subject: [PATCH 5/8] Compact the replication queue from _unref so cancelled batches do not linger Review feedback: compaction waited for a later _queueBlock call, so a batch of requests that all timed out together (or a closing session) stayed in _queued until some future read. The amortized single-pass compaction now runs from _unref itself: still O(1) per unreference (one pass per 16+ stale entries), but cleanup no longer depends on later reads or a connected peer. Co-authored-by: generaltony007 <144252170+GeneralTony007@users.noreply.github.com> --- lib/replicator.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/replicator.js b/lib/replicator.js index 335d8dcc..5057140b 100644 --- a/lib/replicator.js +++ b/lib/replicator.js @@ -215,10 +215,17 @@ class BlockRequest extends Attachable { // Also leave the replication queue: nothing else removes the request // while no peer is connected, so it would stay there until the core - // closes. Removal is amortized - _queueBlock compacts the queue in a - // single pass once stale entries dominate, so unreferencing many - // queued requests at once stays linear instead of quadratic. - if (this.queued === true) this.replicator._queuedStale++ + // closes. Removal is amortized: the queue is compacted in a single + // pass once enough stale entries collect, so unreferencing many + // queued requests at once stays linear instead of quadratic, and a + // cancelled batch does not linger waiting for a later read. + if (this.queued === true) { + const r = this.replicator + if (++r._queuedStale > 16 && r._queuedStale * 2 > r._queued.length) { + r._queued = r._queued.filter((b) => b.queued === true) + r._queuedStale = 0 + } + } this.queued = false for (const req of this.inflight) { @@ -2521,13 +2528,6 @@ module.exports = class Replicator { if (b.inflight.length > 0 || b.queued === true) return b.queued = true this._queued.push(b) - - // Stale entries are compacted here in one pass so that each - // unreference is O(1); a sweep with a connected peer drops them too. - if (this._queuedStale > 16 && this._queuedStale * 2 > this._queued.length) { - this._queued = this._queued.filter((b) => b.queued === true) - this._queuedStale = 0 - } } _resolveHashLocally(peer, req) { From 756cc3cc58701410fefb1e6250b5b329c087b34c Mon Sep 17 00:00:00 2001 From: generaltony007 <191973953+hakarrr@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:49:29 +0530 Subject: [PATCH 6/8] Cover concurrent timeout batches in the queue removal test Adds a 40-request batch that all times out at once, so the test also covers cleanup when no later read triggers it. Fails against the previous revision (whole batch retained), passes now. The bound tightens to 16, matching the compaction threshold. Co-authored-by: generaltony007 <144252170+GeneralTony007@users.noreply.github.com> --- test/replicate.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/test/replicate.js b/test/replicate.js index ac1cdb0a..b02c029d 100644 --- a/test/replicate.js +++ b/test/replicate.js @@ -3530,10 +3530,16 @@ test('timed-out block requests are removed from the replication queue', async fu await t.exception(b.get(i, { timeout: 20 }), /REQUEST_TIMEOUT/) } - // Removal is amortized: _queueBlock compacts the queue once more than 16 - // stale entries dominate it, so the queue never retains more than 17 of - // the 100 timed-out requests (previously it retained all of them). - t.ok(b.core.replicator._queued.length <= 17, 'timed-out requests leave the queue') + // A concurrent batch that all times out together must be cleaned up too, + // without any later read to trigger it + const batch = [] + for (let i = 0; i < 40; i++) batch.push(t.exception(b.get(i, { timeout: 20 }), /REQUEST_TIMEOUT/)) + await Promise.all(batch) + + // Removal is amortized: the queue is compacted once more than 16 stale + // entries dominate it, so it never retains more than 16 timed-out + // requests (previously it retained all of them until the core closed). + t.ok(b.core.replicator._queued.length <= 16, 'timed-out requests leave the queue') }) function noop() {} From ab3727988c02adaa2941fa5b6439fc84eeb72cbe Mon Sep 17 00:00:00 2001 From: generaltony007 <191973953+hakarrr@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:57:22 +0530 Subject: [PATCH 7/8] Mark the cancelled request before compacting the queue The compaction pass filtered before the request being cancelled was marked, so that entry survived every pass and went uncounted; sequential timeouts could push the queue one past the bound. Mark it first so the pass drops it too. Co-authored-by: generaltony007 <144252170+GeneralTony007@users.noreply.github.com> --- lib/replicator.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/replicator.js b/lib/replicator.js index 5057140b..5dfa4855 100644 --- a/lib/replicator.js +++ b/lib/replicator.js @@ -218,15 +218,18 @@ class BlockRequest extends Attachable { // closes. Removal is amortized: the queue is compacted in a single // pass once enough stale entries collect, so unreferencing many // queued requests at once stays linear instead of quadratic, and a - // cancelled batch does not linger waiting for a later read. - if (this.queued === true) { + // cancelled batch does not linger waiting for a later read. The current + // request is marked first so the pass can drop it too. + const queued = this.queued + this.queued = false + + if (queued === true) { const r = this.replicator if (++r._queuedStale > 16 && r._queuedStale * 2 > r._queued.length) { r._queued = r._queued.filter((b) => b.queued === true) r._queuedStale = 0 } } - this.queued = false for (const req of this.inflight) { req.peer._cancelRequest(req) From 389a616a67c873c85bb13b8168d915e8559fe833 Mon Sep 17 00:00:00 2001 From: generaltony007 <191973953+hakarrr@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:58:20 +0530 Subject: [PATCH 8/8] Assert the queue bound between compaction passes Tracks the largest the queue grows across the whole run instead of only checking at the end. Sequential timeouts used to peak one past the bound between passes; now covered. Co-authored-by: generaltony007 <144252170+GeneralTony007@users.noreply.github.com> --- test/replicate.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/replicate.js b/test/replicate.js index b02c029d..0b583e1a 100644 --- a/test/replicate.js +++ b/test/replicate.js @@ -3525,9 +3525,13 @@ test('timed-out block requests are removed from the replication queue', async fu const a = await create(t) const b = await create(t, a.key) - // b never connects to a peer, so its requests can only time out + // b never connects to a peer, so its requests can only time out. Track the + // largest the queue gets: the bound must hold between compaction passes, + // not just at the end. + let maxQueued = 0 for (let i = 0; i < 100; i++) { await t.exception(b.get(i, { timeout: 20 }), /REQUEST_TIMEOUT/) + maxQueued = Math.max(maxQueued, b.core.replicator._queued.length) } // A concurrent batch that all times out together must be cleaned up too, @@ -3535,11 +3539,12 @@ test('timed-out block requests are removed from the replication queue', async fu const batch = [] for (let i = 0; i < 40; i++) batch.push(t.exception(b.get(i, { timeout: 20 }), /REQUEST_TIMEOUT/)) await Promise.all(batch) + maxQueued = Math.max(maxQueued, b.core.replicator._queued.length) // Removal is amortized: the queue is compacted once more than 16 stale // entries dominate it, so it never retains more than 16 timed-out // requests (previously it retained all of them until the core closed). - t.ok(b.core.replicator._queued.length <= 16, 'timed-out requests leave the queue') + t.ok(maxQueued <= 16, 'timed-out requests leave the queue') }) function noop() {}