From f019dbdfc78878ee828eab396f6731421a33611a Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 7 Oct 2026 10:45:57 +0100 Subject: [PATCH 01/13] expose flushWAL method --- index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/index.js b/index.js index 2e28113..ba7a88f 100644 --- a/index.js +++ b/index.js @@ -436,6 +436,10 @@ class HypercoreStorage { return new CoreTX(this.core, this.db, this.atom ? this.view : null, []) } + flushWAL() { + return this.db.flushWAL() + } + close() { if (this.view !== null) { this.view.readStop() From 707458a51e3ae2ca12f01b6a278efab7d6153dc3 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Thu, 8 Oct 2026 15:07:28 +0100 Subject: [PATCH 02/13] add fsyncsNeeded field to core head --- build.js | 4 +++ encoding/spec/hyperschema/index.js | 52 ++++++++++++++------------- encoding/spec/hyperschema/schema.json | 14 +++++++- 3 files changed, 44 insertions(+), 26 deletions(-) diff --git a/build.js b/build.js index 0b6cd9f..5dda2f8 100644 --- a/build.js +++ b/build.js @@ -337,6 +337,10 @@ core.register({ name: 'timestamp', type: 'uint64', required: false + }, + { + name: 'fsyncsNeeded', + type: 'uint' } ] }) diff --git a/encoding/spec/hyperschema/index.js b/encoding/spec/hyperschema/index.js index 5de47d4..fb4f786 100644 --- a/encoding/spec/hyperschema/index.js +++ b/encoding/spec/hyperschema/index.js @@ -1,5 +1,5 @@ // This file is autogenerated by the hyperschema compiler -// Schema Version: 2 +// Schema Version: 3 /* eslint-disable camelcase */ /* eslint-disable quotes */ /* eslint-disable space-before-function-paren */ @@ -7,7 +7,7 @@ const { c } = require('hyperschema/runtime') const external0 = require('../../external.js') -const VERSION = 2 +const VERSION = 3 // eslint-disable-next-line no-unused-vars let version = VERSION @@ -52,11 +52,10 @@ const encoding1 = { if (m.defaultDiscoveryKey) c.fixed32.encode(state, m.defaultDiscoveryKey) }, decode(state) { - const v = c.uint.decode(state) const flags = c.uint.decode(state) return { - version: v, + version: 0, allocated: (flags & 1) !== 0 ? encoding0.decode(state) : null, seed: (flags & 2) !== 0 ? c.fixed32.decode(state) : null, defaultDiscoveryKey: (flags & 4) !== 0 ? c.fixed32.decode(state) : null @@ -87,14 +86,13 @@ const encoding2 = { if (m.defaultDiscoveryKey) c.fixed32.encode(state, m.defaultDiscoveryKey) }, decode(state) { - const v = c.uint.decode(state) const r0 = c.uint.decode(state) const r1 = c.uint.decode(state) const r2 = c.uint.decode(state) const flags = c.uint.decode(state) return { - version: v, + version: 0, cores: r0, datas: r1, groups: r2, @@ -107,8 +105,9 @@ const encoding2 = { // @corestore/head const encoding3 = { preencode(state, m) { - c.uint.preencode(state, m.version) - switch (m.version) { + const v = m.version ?? 2 + c.uint.preencode(state, v) + switch (v) { case 0: case 1: encoding1.preencode(state, m) @@ -121,8 +120,9 @@ const encoding3 = { } }, encode(state, m) { - c.uint.encode(state, m.version) - switch (m.version) { + const v = m.version ?? 2 + c.uint.encode(state, v) + switch (v) { case 0: case 1: encoding1.encode(state, m) @@ -135,18 +135,18 @@ const encoding3 = { } }, decode(state) { - const start = state.start const v = c.uint.decode(state) - state.start = start switch (v) { case 0: case 1: { const decoded = encoding1.decode(state) + decoded.version = v const map = external0.headLegacyMap return map(decoded) } case 2: { const decoded = encoding2.decode(state) + decoded.version = v return decoded } default: @@ -217,10 +217,10 @@ const encoding6_enum = { // @core/hashes enum const encoding6 = { - preencode(state, m) { + preencode (state, m) { state.end++ // max enum is 0 so always one byte }, - encode(state, m) { + encode (state, m) { switch (m) { case 'blake2b': c.uint.encode(state, 0) @@ -229,12 +229,11 @@ const encoding6 = { throw new Error('Unknown enum') } }, - decode(state) { + decode (state) { switch (c.uint.decode(state)) { case 0: return 'blake2b' - default: - return null + default: return null } } } @@ -245,10 +244,10 @@ const encoding7_enum = { // @core/signatures enum const encoding7 = { - preencode(state, m) { + preencode (state, m) { state.end++ // max enum is 0 so always one byte }, - encode(state, m) { + encode (state, m) { switch (m) { case 'ed25519': c.uint.encode(state, 0) @@ -257,12 +256,11 @@ const encoding7 = { throw new Error('Unknown enum') } }, - decode(state) { + decode (state) { switch (c.uint.decode(state)) { case 0: return 'ed25519' - default: - return null + default: return null } } } @@ -481,12 +479,14 @@ const encoding15 = { c.uint.preencode(state, m.length) c.fixed32.preencode(state, m.rootHash) c.optionalBuffer.preencode(state, m.signature) - state.end++ // max flag is 1 so always one byte + state.end++ // max flag is 2 so always one byte if (version >= 2 && m.timestamp) c.uint64.preencode(state, m.timestamp) + if (version >= 3 && m.fsyncsNeeded) c.uint.preencode(state, m.fsyncsNeeded) }, encode(state, m) { - const flags = version >= 2 && m.timestamp ? 1 : 0 + const flags = + ((version >= 2 && m.timestamp) ? 1 : 0) | ((version >= 3 && m.fsyncsNeeded) ? 2 : 0) c.uint.encode(state, m.fork) c.uint.encode(state, m.length) @@ -495,6 +495,7 @@ const encoding15 = { c.uint.encode(state, flags) if (version >= 2 && m.timestamp) c.uint64.encode(state, m.timestamp) + if (version >= 3 && m.fsyncsNeeded) c.uint.encode(state, m.fsyncsNeeded) }, decode(state) { const r0 = c.uint.decode(state) @@ -508,7 +509,8 @@ const encoding15 = { length: r1, rootHash: r2, signature: r3, - timestamp: version >= 2 && (flags & 1) !== 0 ? c.uint64.decode(state) : 0 + timestamp: (version >= 2 && (flags & 1) !== 0) ? c.uint64.decode(state) : 0, + fsyncsNeeded: (version >= 3 && (flags & 2) !== 0) ? c.uint.decode(state) : 0 } } } diff --git a/encoding/spec/hyperschema/schema.json b/encoding/spec/hyperschema/schema.json index 61741d3..28c7b2b 100644 --- a/encoding/spec/hyperschema/schema.json +++ b/encoding/spec/hyperschema/schema.json @@ -1,5 +1,5 @@ { - "version": 2, + "version": 3, "schema": [ { "name": "allocated", @@ -83,6 +83,7 @@ { "name": "head", "namespace": "corestore", + "framed": false, "versions": [ { "type": "@corestore/head-v1", @@ -405,6 +406,11 @@ "required": false, "type": "uint64", "version": 2 + }, + { + "name": "fsyncsNeeded", + "type": "uint", + "version": 3 } ] }, @@ -477,5 +483,11 @@ } ] } + ], + "namespaces": [ + { + "name": "corestore", + "external": "../../external.js" + } ] } From cb55bbab6e7377874ca72e2d644a281991406c30 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Thu, 8 Oct 2026 15:08:40 +0100 Subject: [PATCH 03/13] add debounced fsync --- index.js | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++-- lib/keys.js | 10 +++++- lib/tx.js | 9 +++++ package.json | 1 + 4 files changed, 110 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index ba7a88f..78a42a4 100644 --- a/index.js +++ b/index.js @@ -5,11 +5,13 @@ const DeviceFile = require('device-file') const path = require('path') const fs = require('fs') const Xache = require('xache') +const assert = require('nanoassert') const View = require('./lib/view.js') const VERSION = 2 const COLUMN_FAMILY = 'corestore' +const MIN_FSYNC_GAP = 25 const { store, core } = require('./lib/keys.js') @@ -436,8 +438,13 @@ class HypercoreStorage { return new CoreTX(this.core, this.db, this.atom ? this.view : null, []) } - flushWAL() { - return this.db.flushWAL() + fsyncsNeeded() { + assert(this.store.fsyncs !== -1, 'Storage not opened') + return this.store.fsyncsStarted + 1 + } + + fsync(target) { + return this.store.flushFsync(target) } close() { @@ -516,6 +523,13 @@ class CorestoreStorage { this.version = 0 this.migrating = null + // fsync state + this.flock = new ScopeLock({ debounce: true }) + this.fsyncing = null + this.fsyncs = -1 + this.fsyncsStarted = 0 + this.lastFsyncAt = 0 + if ((this.bootstrap && !this.readOnly && !this.allowBackup) || this.wait) { const corestoreFile = path.join(this.path, 'CORESTORE') @@ -553,13 +567,79 @@ class CorestoreStorage { async ready() { if (this.version === 0) await this._migrateStore() - return this.db.ready() + await this.db.ready() + } + + async _openFsyncs() { + this.fsyncs = (await this._getFsyncs()) || 0 + this.fsyncsStarted = this.fsyncs } compact(opts) { return this.db.compactRange(opts) } + async flushFsync(target) { + while (this.fsyncs < target) { + if (this.db.closed) throw new Error('Storage has closed') + await this.fsync(target) + } + } + + async fsync(target) { + if (!(await this.flock.lock())) { + return // debounced + } + + if (this.fsyncs >= target) { + this.flock.unlock() + return + } + + this.fsyncing = this._fsync() + const current = this.fsyncing + + try { + await this.fsyncing + } finally { + this.fsyncing = null + this.flock.unlock() + } + + return current + } + + async _fsync() { + const gap = MIN_FSYNC_GAP - (Date.now() - this.lastFsyncAt) + if (gap > 0) await sleep(gap) + + const gen = ++this.fsyncsStarted + await this.db.flushWAL() + this.lastFsyncAt = Date.now() + + await this._setFsyncs(gen) + + this.fsyncs = gen + + return gen + } + + async _setFsyncs(gen) { + // enter requires lock, we are under flock so write directly + const tx = this.db.write({ autoDestroy: true }) + CorestoreTX.setFsyncs(tx, gen) + + await tx.flush() + } + + async _getFsyncs() { + const rx = new CorestoreRX(this.db, EMPTY) + const fsyncsPromise = rx.getFsyncs() + rx.tryFlush() + + return fsyncsPromise + } + async audit() { for await (const { core } of this.createCoreStream()) { const coreRx = new CoreRX(core, this.db, EMPTY, this.treeCache, this.stats, -1) @@ -667,9 +747,13 @@ class CorestoreStorage { const rx = new CorestoreRX(this.db, view) const headPromise = rx.getHead() + const fsyncsPromise = rx.getFsyncs() rx.tryFlush() const head = await headPromise + const fsyncs = (await fsyncsPromise) || 0 + + this.fsyncs = this.fsyncsStarted = fsyncs const version = head === null ? 0 : head.version if (version === VERSION) { @@ -1407,3 +1491,7 @@ async function getInfoFromBatch(db, c, discoveryKey, getAuth, getHead, getHints) hints: await hintsPromise } } + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} diff --git a/lib/keys.js b/lib/keys.js index 2ee4a15..a2ddd15 100644 --- a/lib/keys.js +++ b/lib/keys.js @@ -9,8 +9,9 @@ const TL_CORE = 3 const TL_DATA = 4 const TL_GROUP_BY_TOPIC = 5 const TL_GROUP = 6 +const TL_FSYNC = 7 -const TL_END = TL_GROUP + 1 +const TL_END = TL_FSYNC + 1 const CORE_AUTH = 0 const CORE_SESSIONS = 1 @@ -118,6 +119,13 @@ store.discoveryKey = function (buffer) { return c.fixed32.decode(state) } +store.fsync = function () { + const state = alloc() + const start = state.start + UINT.encode(state, TL_FSYNC) + return state.buffer.subarray(start, state.start) +} + core.core = function (ptr) { const state = alloc() const start = state.start diff --git a/lib/tx.js b/lib/tx.js index f619154..64316c1 100644 --- a/lib/tx.js +++ b/lib/tx.js @@ -9,6 +9,7 @@ const View = require('./view.js') const CORESTORE_HEAD = schema.getEncoding('@corestore/head') const CORESTORE_CORE = schema.getEncoding('@corestore/core') const CORESTORE_GROUP = c.uint +const CORESTORE_FSYNC = c.uint const CORE_AUTH = schema.getEncoding('@core/auth') const CORE_GROUP = schema.getEncoding('@core/group') @@ -365,6 +366,10 @@ class CorestoreTX { this.changes.push([group.topic(topic), null, null]) } + static setFsyncs(tx, n) { + return tx.put(store.fsync(), encode(CORESTORE_FSYNC, n)) + } + clear() { const [start, end] = store.clear() this.changes.push([start, null, end]) @@ -405,6 +410,10 @@ class CorestoreRX { return decode(CORESTORE_GROUP, await this.view.get(this.read, group.topic(topic))) } + async getFsyncs() { + return decode(CORESTORE_FSYNC, await this.view.get(this.read, store.fsync())) + } + tryFlush() { this.read.tryFlush() this._free() diff --git a/package.json b/package.json index f14b2e6..a52702e 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "hypercore-crypto": "^3.4.2", "hyperschema": "^1.21.0", "index-encoder": "^3.3.2", + "nanoassert": "^3.1.0", "resolve-reject-promise": "^1.0.0", "rocksdb-native": "^3.18.0", "scope-lock": "^1.2.4", From 2286bdea0f72f02c7637f9e904e640647fd48646 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Thu, 8 Oct 2026 15:08:56 +0100 Subject: [PATCH 04/13] add fsync tests --- test/all.js | 1 + test/atomic.js | 3 +- test/basic.js | 9 +++++- test/core.js | 6 ++-- test/fsync.js | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 test/fsync.js diff --git a/test/all.js b/test/all.js index 7d4b916..7dd0eec 100644 --- a/test/all.js +++ b/test/all.js @@ -10,6 +10,7 @@ async function runTests() { await import('./atomic.js') await import('./basic.js') await import('./core.js') + await import('./fsync.js') await import('./groups.js') await import('./snapshot.js') await import('./streams.js') diff --git a/test/atomic.js b/test/atomic.js index 6c4f422..cbfd893 100644 --- a/test/atomic.js +++ b/test/atomic.js @@ -185,7 +185,8 @@ test('atomized flow with all non-delete operations on a single core', async (t) length: 3, rootHash: b4a.from('a'.repeat(64), 'hex'), signature: b4a.from('b'.repeat(64), 'hex'), - timestamp: 0 + timestamp: 0, + fsyncsNeeded: 0 } const expDependency = { dataPointer: 1, diff --git a/test/basic.js b/test/basic.js index de7cf8e..71948be 100644 --- a/test/basic.js +++ b/test/basic.js @@ -263,7 +263,14 @@ test('can get info from store efficiently', async function (t) { keyPair: null, encryptionKey: null }, - head: { fork: 0, length: 2, rootHash: b4a.alloc(32, 0), signature: null, timestamp: 0 }, + head: { + fork: 0, + length: 2, + rootHash: b4a.alloc(32, 0), + signature: null, + timestamp: 0, + fsyncsNeeded: 0 + }, hints: null }) } diff --git a/test/core.js b/test/core.js index 75567b9..080341e 100644 --- a/test/core.js +++ b/test/core.js @@ -360,7 +360,8 @@ test('set and get hypercore head', async (t) => { length: 3, rootHash: b4a.from('a'.repeat(64), 'hex'), signature: b4a.from('b'.repeat(64), 'hex'), - timestamp: 0 + timestamp: 0, + fsyncsNeeded: 0 }, 'updated head' ) @@ -899,7 +900,8 @@ test('export hypercore', async (t) => { length: 3, rootHash: b4a.from('a'.repeat(64), 'hex'), signature: b4a.from('b'.repeat(64), 'hex'), - timestamp: 0 + timestamp: 0, + fsyncsNeeded: 0 } const node0 = { index: 0, size: 1, hash: b4a.from('a'.repeat(64), 'hex') } diff --git a/test/fsync.js b/test/fsync.js new file mode 100644 index 0000000..7f3fccc --- /dev/null +++ b/test/fsync.js @@ -0,0 +1,87 @@ +const test = require('brittle') +const b4a = require('b4a') +const Storage = require('../') +const { create } = require('./helpers') + +test('fsyncs', async function (t) { + const s = await create(t) + await s.ready() + + const a = await s.createCore({ key: b4a.alloc(32, 1), discoveryKey: b4a.alloc(32, 1) }) + + const target = a.fsyncsNeeded() + + t.is(s.fsyncs, 0) + t.is(target, 1) + + await s.flushFsync(target) + + t.is(s.fsyncs, target) + + await a.close() + await s.close() +}) + +test('fsyncs is persisted', async function (t) { + const dir = await t.tmp() + + const s1 = new Storage(dir) + await s1.ready() + + await s1.flushFsync(3) + + t.is(s1.fsyncs, 3) + + await s1.close() + + const s2 = new Storage(dir) + await s2.ready() + + t.is(s2.fsyncs, 3) + + const a = await s2.createCore({ key: b4a.alloc(32, 1), discoveryKey: b4a.alloc(32, 1) }) + + const target = a.fsyncsNeeded() + t.is(target, 4) + + await s2.flushFsync(target) + + t.is(s2.fsyncs, target) + + await a.close() + await s2.close() +}) + +test('fsyncs is debounced', async function (t) { + const dir = await t.tmp() + + const s = new Storage(dir) + await s.ready() + + const a = await s.createCore({ key: b4a.alloc(32, 1), discoveryKey: b4a.alloc(32, 1) }) + const b = await s.createCore({ key: b4a.alloc(32, 2), discoveryKey: b4a.alloc(32, 2) }) + const c = await s.createCore({ key: b4a.alloc(32, 3), discoveryKey: b4a.alloc(32, 3) }) + + const t0 = a.fsyncsNeeded() + const tb = b.fsyncsNeeded() + const tc = c.fsyncsNeeded() + + t.is(t0, tb) + t.is(t0, tc) + + const proms = [] + for (let i = 0; i < 10; i++) { + proms.push(s.flushFsync(t0)) + } + + await Promise.all(proms) + + const t1 = a.fsyncsNeeded() + + // debounced + t.is(s.fsyncs, t0) + t.is(t1, s.fsyncs + 1) + + await a.close() + await s.close() +}) From b320a637f79ca755df1d61ec7d15be1f926954e2 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Thu, 8 Oct 2026 15:15:52 +0100 Subject: [PATCH 05/13] update readme --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 72bbc53..270c2d5 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,15 @@ Check if a core exists. List all cores. Stream data looks like this `{ discoveryKey, core }` where core contains the core header. +#### `await store.flushFsync(target)` + +Wait until the WAL has been fsynced at least up to `target` (a value from `core.fsyncsNeeded()`). +Concurrent callers share the same fsync, which is paced to run at most every 25ms. + +#### `store.fsyncs` + +The latest fsync generation known to be on disk. Persisted and restored on open. + #### `await store.close()` Close the storage instance. @@ -203,6 +212,15 @@ Create a stream of all user data. `opts` is a query object with the following po } ``` +#### `target = core.fsyncsNeeded()` + +Returns the fsync generation that will cover everything written so far. Store it with the head +(`fsyncsNeeded`) and pass it to `core.fsync(target)` once you need the writes to be durable. + +#### `await core.fsync(target)` + +Same as `store.flushFsync(target)`. + #### `await core.close()` Close the core storage engine. From 93a32fa32a9256647fcac0f8bd8196e34cc164c5 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Thu, 8 Oct 2026 18:16:24 +0100 Subject: [PATCH 06/13] debounce separately and lock only when writing head --- build.js | 4 ++ encoding/spec/hyperschema/index.js | 35 ++++++++------ encoding/spec/hyperschema/schema.json | 5 ++ index.js | 68 ++++++++++----------------- lib/keys.js | 10 +--- lib/tx.js | 9 ---- 6 files changed, 55 insertions(+), 76 deletions(-) diff --git a/build.js b/build.js index 5dda2f8..6345faf 100644 --- a/build.js +++ b/build.js @@ -68,6 +68,10 @@ corestore.register({ { name: 'defaultDiscoveryKey', type: 'fixed32' + }, + { + name: 'fsyncs', + type: 'uint' } ] }) diff --git a/encoding/spec/hyperschema/index.js b/encoding/spec/hyperschema/index.js index fb4f786..ca03c07 100644 --- a/encoding/spec/hyperschema/index.js +++ b/encoding/spec/hyperschema/index.js @@ -69,13 +69,15 @@ const encoding2 = { c.uint.preencode(state, m.cores) c.uint.preencode(state, m.datas) c.uint.preencode(state, m.groups) - state.end++ // max flag is 2 so always one byte + state.end++ // max flag is 4 so always one byte if (m.seed) c.fixed32.preencode(state, m.seed) if (m.defaultDiscoveryKey) c.fixed32.preencode(state, m.defaultDiscoveryKey) + if (version >= 3 && m.fsyncs) c.uint.preencode(state, m.fsyncs) }, encode(state, m) { - const flags = (m.seed ? 1 : 0) | (m.defaultDiscoveryKey ? 2 : 0) + const flags = + (m.seed ? 1 : 0) | (m.defaultDiscoveryKey ? 2 : 0) | (version >= 3 && m.fsyncs ? 4 : 0) c.uint.encode(state, m.cores) c.uint.encode(state, m.datas) @@ -84,6 +86,7 @@ const encoding2 = { if (m.seed) c.fixed32.encode(state, m.seed) if (m.defaultDiscoveryKey) c.fixed32.encode(state, m.defaultDiscoveryKey) + if (version >= 3 && m.fsyncs) c.uint.encode(state, m.fsyncs) }, decode(state) { const r0 = c.uint.decode(state) @@ -97,7 +100,8 @@ const encoding2 = { datas: r1, groups: r2, seed: (flags & 1) !== 0 ? c.fixed32.decode(state) : null, - defaultDiscoveryKey: (flags & 2) !== 0 ? c.fixed32.decode(state) : null + defaultDiscoveryKey: (flags & 2) !== 0 ? c.fixed32.decode(state) : null, + fsyncs: version >= 3 && (flags & 4) !== 0 ? c.uint.decode(state) : 0 } } } @@ -217,10 +221,10 @@ const encoding6_enum = { // @core/hashes enum const encoding6 = { - preencode (state, m) { + preencode(state, m) { state.end++ // max enum is 0 so always one byte }, - encode (state, m) { + encode(state, m) { switch (m) { case 'blake2b': c.uint.encode(state, 0) @@ -229,11 +233,12 @@ const encoding6 = { throw new Error('Unknown enum') } }, - decode (state) { + decode(state) { switch (c.uint.decode(state)) { case 0: return 'blake2b' - default: return null + default: + return null } } } @@ -244,10 +249,10 @@ const encoding7_enum = { // @core/signatures enum const encoding7 = { - preencode (state, m) { + preencode(state, m) { state.end++ // max enum is 0 so always one byte }, - encode (state, m) { + encode(state, m) { switch (m) { case 'ed25519': c.uint.encode(state, 0) @@ -256,11 +261,12 @@ const encoding7 = { throw new Error('Unknown enum') } }, - decode (state) { + decode(state) { switch (c.uint.decode(state)) { case 0: return 'ed25519' - default: return null + default: + return null } } } @@ -485,8 +491,7 @@ const encoding15 = { if (version >= 3 && m.fsyncsNeeded) c.uint.preencode(state, m.fsyncsNeeded) }, encode(state, m) { - const flags = - ((version >= 2 && m.timestamp) ? 1 : 0) | ((version >= 3 && m.fsyncsNeeded) ? 2 : 0) + const flags = (version >= 2 && m.timestamp ? 1 : 0) | (version >= 3 && m.fsyncsNeeded ? 2 : 0) c.uint.encode(state, m.fork) c.uint.encode(state, m.length) @@ -509,8 +514,8 @@ const encoding15 = { length: r1, rootHash: r2, signature: r3, - timestamp: (version >= 2 && (flags & 1) !== 0) ? c.uint64.decode(state) : 0, - fsyncsNeeded: (version >= 3 && (flags & 2) !== 0) ? c.uint.decode(state) : 0 + timestamp: version >= 2 && (flags & 1) !== 0 ? c.uint64.decode(state) : 0, + fsyncsNeeded: version >= 3 && (flags & 2) !== 0 ? c.uint.decode(state) : 0 } } } diff --git a/encoding/spec/hyperschema/schema.json b/encoding/spec/hyperschema/schema.json index 28c7b2b..1614efb 100644 --- a/encoding/spec/hyperschema/schema.json +++ b/encoding/spec/hyperschema/schema.json @@ -77,6 +77,11 @@ "name": "defaultDiscoveryKey", "type": "fixed32", "version": 1 + }, + { + "name": "fsyncs", + "type": "uint", + "version": 3 } ] }, diff --git a/index.js b/index.js index 78a42a4..dcea09f 100644 --- a/index.js +++ b/index.js @@ -524,10 +524,9 @@ class CorestoreStorage { this.migrating = null // fsync state - this.flock = new ScopeLock({ debounce: true }) - this.fsyncing = null this.fsyncs = -1 this.fsyncsStarted = 0 + this.fsyncing = null this.lastFsyncAt = 0 if ((this.bootstrap && !this.readOnly && !this.allowBackup) || this.wait) { @@ -570,11 +569,6 @@ class CorestoreStorage { await this.db.ready() } - async _openFsyncs() { - this.fsyncs = (await this._getFsyncs()) || 0 - this.fsyncsStarted = this.fsyncs - } - compact(opts) { return this.db.compactRange(opts) } @@ -587,57 +581,46 @@ class CorestoreStorage { } async fsync(target) { - if (!(await this.flock.lock())) { - return // debounced - } - - if (this.fsyncs >= target) { - this.flock.unlock() + if (this.fsyncing !== null) { + if (target <= this.fsyncing.gen) return this.fsyncing.promise + await this.fsyncing.promise.catch(noop) return } - this.fsyncing = this._fsync() - const current = this.fsyncing + const gen = this.fsyncsStarted + 1 + const promise = this._fsync(gen) + this.fsyncing = { gen, promise } try { - await this.fsyncing + await promise } finally { this.fsyncing = null - this.flock.unlock() } - - return current } - async _fsync() { + async _fsync(gen) { const gap = MIN_FSYNC_GAP - (Date.now() - this.lastFsyncAt) if (gap > 0) await sleep(gap) - const gen = ++this.fsyncsStarted + this.fsyncsStarted = gen await this.db.flushWAL() - this.lastFsyncAt = Date.now() - - await this._setFsyncs(gen) - - this.fsyncs = gen - return gen - } + this.lastFsyncAt = Date.now() - async _setFsyncs(gen) { - // enter requires lock, we are under flock so write directly - const tx = this.db.write({ autoDestroy: true }) - CorestoreTX.setFsyncs(tx, gen) + const view = await this._enter() + try { + const head = await this._getHead(view) + const tx = new CorestoreTX(view) - await tx.flush() - } + head.fsyncs = gen - async _getFsyncs() { - const rx = new CorestoreRX(this.db, EMPTY) - const fsyncsPromise = rx.getFsyncs() - rx.tryFlush() + tx.setHead(head) + tx.apply() + } finally { + await this._exit() + } - return fsyncsPromise + this.fsyncs = gen } async audit() { @@ -747,13 +730,11 @@ class CorestoreStorage { const rx = new CorestoreRX(this.db, view) const headPromise = rx.getHead() - const fsyncsPromise = rx.getFsyncs() rx.tryFlush() const head = await headPromise - const fsyncs = (await fsyncsPromise) || 0 - this.fsyncs = this.fsyncsStarted = fsyncs + this.fsyncs = this.fsyncsStarted = head ? head.fsyncs : 0 const version = head === null ? 0 : head.version if (version === VERSION) { @@ -1347,7 +1328,8 @@ function initStoreHead() { cores: 0, groups: 0, seed: null, - defaultDiscoveryKey: null + defaultDiscoveryKey: null, + fsyncs: 0 } } diff --git a/lib/keys.js b/lib/keys.js index a2ddd15..2ee4a15 100644 --- a/lib/keys.js +++ b/lib/keys.js @@ -9,9 +9,8 @@ const TL_CORE = 3 const TL_DATA = 4 const TL_GROUP_BY_TOPIC = 5 const TL_GROUP = 6 -const TL_FSYNC = 7 -const TL_END = TL_FSYNC + 1 +const TL_END = TL_GROUP + 1 const CORE_AUTH = 0 const CORE_SESSIONS = 1 @@ -119,13 +118,6 @@ store.discoveryKey = function (buffer) { return c.fixed32.decode(state) } -store.fsync = function () { - const state = alloc() - const start = state.start - UINT.encode(state, TL_FSYNC) - return state.buffer.subarray(start, state.start) -} - core.core = function (ptr) { const state = alloc() const start = state.start diff --git a/lib/tx.js b/lib/tx.js index 64316c1..f619154 100644 --- a/lib/tx.js +++ b/lib/tx.js @@ -9,7 +9,6 @@ const View = require('./view.js') const CORESTORE_HEAD = schema.getEncoding('@corestore/head') const CORESTORE_CORE = schema.getEncoding('@corestore/core') const CORESTORE_GROUP = c.uint -const CORESTORE_FSYNC = c.uint const CORE_AUTH = schema.getEncoding('@core/auth') const CORE_GROUP = schema.getEncoding('@core/group') @@ -366,10 +365,6 @@ class CorestoreTX { this.changes.push([group.topic(topic), null, null]) } - static setFsyncs(tx, n) { - return tx.put(store.fsync(), encode(CORESTORE_FSYNC, n)) - } - clear() { const [start, end] = store.clear() this.changes.push([start, null, end]) @@ -410,10 +405,6 @@ class CorestoreRX { return decode(CORESTORE_GROUP, await this.view.get(this.read, group.topic(topic))) } - async getFsyncs() { - return decode(CORESTORE_FSYNC, await this.view.get(this.read, store.fsync())) - } - tryFlush() { this.read.tryFlush() this._free() From b3fa9e938ccb6677f3cd5822d9e8e6020add2d01 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Fri, 9 Oct 2026 12:20:59 +0100 Subject: [PATCH 07/13] fsync can be in memory only --- build.js | 8 ---- encoding/spec/hyperschema/index.js | 43 ++++++++---------- encoding/spec/hyperschema/schema.json | 19 +------- index.js | 22 +--------- test/fsync.js | 63 ++++++++------------------- 5 files changed, 38 insertions(+), 117 deletions(-) diff --git a/build.js b/build.js index 6345faf..0b6cd9f 100644 --- a/build.js +++ b/build.js @@ -68,10 +68,6 @@ corestore.register({ { name: 'defaultDiscoveryKey', type: 'fixed32' - }, - { - name: 'fsyncs', - type: 'uint' } ] }) @@ -341,10 +337,6 @@ core.register({ name: 'timestamp', type: 'uint64', required: false - }, - { - name: 'fsyncsNeeded', - type: 'uint' } ] }) diff --git a/encoding/spec/hyperschema/index.js b/encoding/spec/hyperschema/index.js index ca03c07..5de47d4 100644 --- a/encoding/spec/hyperschema/index.js +++ b/encoding/spec/hyperschema/index.js @@ -1,5 +1,5 @@ // This file is autogenerated by the hyperschema compiler -// Schema Version: 3 +// Schema Version: 2 /* eslint-disable camelcase */ /* eslint-disable quotes */ /* eslint-disable space-before-function-paren */ @@ -7,7 +7,7 @@ const { c } = require('hyperschema/runtime') const external0 = require('../../external.js') -const VERSION = 3 +const VERSION = 2 // eslint-disable-next-line no-unused-vars let version = VERSION @@ -52,10 +52,11 @@ const encoding1 = { if (m.defaultDiscoveryKey) c.fixed32.encode(state, m.defaultDiscoveryKey) }, decode(state) { + const v = c.uint.decode(state) const flags = c.uint.decode(state) return { - version: 0, + version: v, allocated: (flags & 1) !== 0 ? encoding0.decode(state) : null, seed: (flags & 2) !== 0 ? c.fixed32.decode(state) : null, defaultDiscoveryKey: (flags & 4) !== 0 ? c.fixed32.decode(state) : null @@ -69,15 +70,13 @@ const encoding2 = { c.uint.preencode(state, m.cores) c.uint.preencode(state, m.datas) c.uint.preencode(state, m.groups) - state.end++ // max flag is 4 so always one byte + state.end++ // max flag is 2 so always one byte if (m.seed) c.fixed32.preencode(state, m.seed) if (m.defaultDiscoveryKey) c.fixed32.preencode(state, m.defaultDiscoveryKey) - if (version >= 3 && m.fsyncs) c.uint.preencode(state, m.fsyncs) }, encode(state, m) { - const flags = - (m.seed ? 1 : 0) | (m.defaultDiscoveryKey ? 2 : 0) | (version >= 3 && m.fsyncs ? 4 : 0) + const flags = (m.seed ? 1 : 0) | (m.defaultDiscoveryKey ? 2 : 0) c.uint.encode(state, m.cores) c.uint.encode(state, m.datas) @@ -86,22 +85,21 @@ const encoding2 = { if (m.seed) c.fixed32.encode(state, m.seed) if (m.defaultDiscoveryKey) c.fixed32.encode(state, m.defaultDiscoveryKey) - if (version >= 3 && m.fsyncs) c.uint.encode(state, m.fsyncs) }, decode(state) { + const v = c.uint.decode(state) const r0 = c.uint.decode(state) const r1 = c.uint.decode(state) const r2 = c.uint.decode(state) const flags = c.uint.decode(state) return { - version: 0, + version: v, cores: r0, datas: r1, groups: r2, seed: (flags & 1) !== 0 ? c.fixed32.decode(state) : null, - defaultDiscoveryKey: (flags & 2) !== 0 ? c.fixed32.decode(state) : null, - fsyncs: version >= 3 && (flags & 4) !== 0 ? c.uint.decode(state) : 0 + defaultDiscoveryKey: (flags & 2) !== 0 ? c.fixed32.decode(state) : null } } } @@ -109,9 +107,8 @@ const encoding2 = { // @corestore/head const encoding3 = { preencode(state, m) { - const v = m.version ?? 2 - c.uint.preencode(state, v) - switch (v) { + c.uint.preencode(state, m.version) + switch (m.version) { case 0: case 1: encoding1.preencode(state, m) @@ -124,9 +121,8 @@ const encoding3 = { } }, encode(state, m) { - const v = m.version ?? 2 - c.uint.encode(state, v) - switch (v) { + c.uint.encode(state, m.version) + switch (m.version) { case 0: case 1: encoding1.encode(state, m) @@ -139,18 +135,18 @@ const encoding3 = { } }, decode(state) { + const start = state.start const v = c.uint.decode(state) + state.start = start switch (v) { case 0: case 1: { const decoded = encoding1.decode(state) - decoded.version = v const map = external0.headLegacyMap return map(decoded) } case 2: { const decoded = encoding2.decode(state) - decoded.version = v return decoded } default: @@ -485,13 +481,12 @@ const encoding15 = { c.uint.preencode(state, m.length) c.fixed32.preencode(state, m.rootHash) c.optionalBuffer.preencode(state, m.signature) - state.end++ // max flag is 2 so always one byte + state.end++ // max flag is 1 so always one byte if (version >= 2 && m.timestamp) c.uint64.preencode(state, m.timestamp) - if (version >= 3 && m.fsyncsNeeded) c.uint.preencode(state, m.fsyncsNeeded) }, encode(state, m) { - const flags = (version >= 2 && m.timestamp ? 1 : 0) | (version >= 3 && m.fsyncsNeeded ? 2 : 0) + const flags = version >= 2 && m.timestamp ? 1 : 0 c.uint.encode(state, m.fork) c.uint.encode(state, m.length) @@ -500,7 +495,6 @@ const encoding15 = { c.uint.encode(state, flags) if (version >= 2 && m.timestamp) c.uint64.encode(state, m.timestamp) - if (version >= 3 && m.fsyncsNeeded) c.uint.encode(state, m.fsyncsNeeded) }, decode(state) { const r0 = c.uint.decode(state) @@ -514,8 +508,7 @@ const encoding15 = { length: r1, rootHash: r2, signature: r3, - timestamp: version >= 2 && (flags & 1) !== 0 ? c.uint64.decode(state) : 0, - fsyncsNeeded: version >= 3 && (flags & 2) !== 0 ? c.uint.decode(state) : 0 + timestamp: version >= 2 && (flags & 1) !== 0 ? c.uint64.decode(state) : 0 } } } diff --git a/encoding/spec/hyperschema/schema.json b/encoding/spec/hyperschema/schema.json index 1614efb..61741d3 100644 --- a/encoding/spec/hyperschema/schema.json +++ b/encoding/spec/hyperschema/schema.json @@ -1,5 +1,5 @@ { - "version": 3, + "version": 2, "schema": [ { "name": "allocated", @@ -77,18 +77,12 @@ "name": "defaultDiscoveryKey", "type": "fixed32", "version": 1 - }, - { - "name": "fsyncs", - "type": "uint", - "version": 3 } ] }, { "name": "head", "namespace": "corestore", - "framed": false, "versions": [ { "type": "@corestore/head-v1", @@ -411,11 +405,6 @@ "required": false, "type": "uint64", "version": 2 - }, - { - "name": "fsyncsNeeded", - "type": "uint", - "version": 3 } ] }, @@ -488,11 +477,5 @@ } ] } - ], - "namespaces": [ - { - "name": "corestore", - "external": "../../external.js" - } ] } diff --git a/index.js b/index.js index dcea09f..a9e1ea4 100644 --- a/index.js +++ b/index.js @@ -439,7 +439,6 @@ class HypercoreStorage { } fsyncsNeeded() { - assert(this.store.fsyncs !== -1, 'Storage not opened') return this.store.fsyncsStarted + 1 } @@ -524,7 +523,7 @@ class CorestoreStorage { this.migrating = null // fsync state - this.fsyncs = -1 + this.fsyncs = 0 this.fsyncsStarted = 0 this.fsyncing = null this.lastFsyncAt = 0 @@ -606,20 +605,6 @@ class CorestoreStorage { await this.db.flushWAL() this.lastFsyncAt = Date.now() - - const view = await this._enter() - try { - const head = await this._getHead(view) - const tx = new CorestoreTX(view) - - head.fsyncs = gen - - tx.setHead(head) - tx.apply() - } finally { - await this._exit() - } - this.fsyncs = gen } @@ -734,8 +719,6 @@ class CorestoreStorage { rx.tryFlush() const head = await headPromise - this.fsyncs = this.fsyncsStarted = head ? head.fsyncs : 0 - const version = head === null ? 0 : head.version if (version === VERSION) { this.version = VERSION @@ -1328,8 +1311,7 @@ function initStoreHead() { cores: 0, groups: 0, seed: null, - defaultDiscoveryKey: null, - fsyncs: 0 + defaultDiscoveryKey: null } } diff --git a/test/fsync.js b/test/fsync.js index 7f3fccc..524a7a9 100644 --- a/test/fsync.js +++ b/test/fsync.js @@ -9,7 +9,7 @@ test('fsyncs', async function (t) { const a = await s.createCore({ key: b4a.alloc(32, 1), discoveryKey: b4a.alloc(32, 1) }) - const target = a.fsyncsNeeded() + const target = a.store.fsyncsStarted + 1 t.is(s.fsyncs, 0) t.is(target, 1) @@ -22,36 +22,6 @@ test('fsyncs', async function (t) { await s.close() }) -test('fsyncs is persisted', async function (t) { - const dir = await t.tmp() - - const s1 = new Storage(dir) - await s1.ready() - - await s1.flushFsync(3) - - t.is(s1.fsyncs, 3) - - await s1.close() - - const s2 = new Storage(dir) - await s2.ready() - - t.is(s2.fsyncs, 3) - - const a = await s2.createCore({ key: b4a.alloc(32, 1), discoveryKey: b4a.alloc(32, 1) }) - - const target = a.fsyncsNeeded() - t.is(target, 4) - - await s2.flushFsync(target) - - t.is(s2.fsyncs, target) - - await a.close() - await s2.close() -}) - test('fsyncs is debounced', async function (t) { const dir = await t.tmp() @@ -62,25 +32,26 @@ test('fsyncs is debounced', async function (t) { const b = await s.createCore({ key: b4a.alloc(32, 2), discoveryKey: b4a.alloc(32, 2) }) const c = await s.createCore({ key: b4a.alloc(32, 3), discoveryKey: b4a.alloc(32, 3) }) - const t0 = a.fsyncsNeeded() - const tb = b.fsyncsNeeded() - const tc = c.fsyncsNeeded() - - t.is(t0, tb) - t.is(t0, tc) - - const proms = [] - for (let i = 0; i < 10; i++) { - proms.push(s.flushFsync(t0)) - } + const proms = [ + a.ensureFsynced(), + a.ensureFsynced(), + a.ensureFsynced(), + b.ensureFsynced(), + b.ensureFsynced(), + b.ensureFsynced(), + c.ensureFsynced(), + c.ensureFsynced(), + c.ensureFsynced() + ] await Promise.all(proms) - const t1 = a.fsyncsNeeded() - // debounced - t.is(s.fsyncs, t0) - t.is(t1, s.fsyncs + 1) + t.is(s.fsyncs, 2) + + await a.ensureFsynced() + + t.is(s.fsyncs, 3) await a.close() await s.close() From 2ca962e693d187f1fd67c45669ec8ce956984893 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Fri, 9 Oct 2026 12:23:38 +0100 Subject: [PATCH 08/13] remove nanoassert --- index.js | 1 - package.json | 1 - 2 files changed, 2 deletions(-) diff --git a/index.js b/index.js index a9e1ea4..1f0f310 100644 --- a/index.js +++ b/index.js @@ -5,7 +5,6 @@ const DeviceFile = require('device-file') const path = require('path') const fs = require('fs') const Xache = require('xache') -const assert = require('nanoassert') const View = require('./lib/view.js') diff --git a/package.json b/package.json index a52702e..f14b2e6 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,6 @@ "hypercore-crypto": "^3.4.2", "hyperschema": "^1.21.0", "index-encoder": "^3.3.2", - "nanoassert": "^3.1.0", "resolve-reject-promise": "^1.0.0", "rocksdb-native": "^3.18.0", "scope-lock": "^1.2.4", From 614f07c968ac460a1e6d1fa50e4d6c281587d5d2 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Fri, 9 Oct 2026 12:28:51 +0100 Subject: [PATCH 09/13] update tests --- index.js | 2 +- test/atomic.js | 3 +-- test/basic.js | 3 +-- test/core.js | 6 ++---- test/fsync.js | 24 ++++++++++++------------ 5 files changed, 17 insertions(+), 21 deletions(-) diff --git a/index.js b/index.js index 1f0f310..98e91c9 100644 --- a/index.js +++ b/index.js @@ -441,7 +441,7 @@ class HypercoreStorage { return this.store.fsyncsStarted + 1 } - fsync(target) { + fsync(target = this.store.fsyncsStarted + 1) { return this.store.flushFsync(target) } diff --git a/test/atomic.js b/test/atomic.js index cbfd893..6c4f422 100644 --- a/test/atomic.js +++ b/test/atomic.js @@ -185,8 +185,7 @@ test('atomized flow with all non-delete operations on a single core', async (t) length: 3, rootHash: b4a.from('a'.repeat(64), 'hex'), signature: b4a.from('b'.repeat(64), 'hex'), - timestamp: 0, - fsyncsNeeded: 0 + timestamp: 0 } const expDependency = { dataPointer: 1, diff --git a/test/basic.js b/test/basic.js index 71948be..e498ac7 100644 --- a/test/basic.js +++ b/test/basic.js @@ -268,8 +268,7 @@ test('can get info from store efficiently', async function (t) { length: 2, rootHash: b4a.alloc(32, 0), signature: null, - timestamp: 0, - fsyncsNeeded: 0 + timestamp: 0 }, hints: null }) diff --git a/test/core.js b/test/core.js index 080341e..75567b9 100644 --- a/test/core.js +++ b/test/core.js @@ -360,8 +360,7 @@ test('set and get hypercore head', async (t) => { length: 3, rootHash: b4a.from('a'.repeat(64), 'hex'), signature: b4a.from('b'.repeat(64), 'hex'), - timestamp: 0, - fsyncsNeeded: 0 + timestamp: 0 }, 'updated head' ) @@ -900,8 +899,7 @@ test('export hypercore', async (t) => { length: 3, rootHash: b4a.from('a'.repeat(64), 'hex'), signature: b4a.from('b'.repeat(64), 'hex'), - timestamp: 0, - fsyncsNeeded: 0 + timestamp: 0 } const node0 = { index: 0, size: 1, hash: b4a.from('a'.repeat(64), 'hex') } diff --git a/test/fsync.js b/test/fsync.js index 524a7a9..f962518 100644 --- a/test/fsync.js +++ b/test/fsync.js @@ -3,7 +3,7 @@ const b4a = require('b4a') const Storage = require('../') const { create } = require('./helpers') -test('fsyncs', async function (t) { +test('fsync', async function (t) { const s = await create(t) await s.ready() @@ -22,7 +22,7 @@ test('fsyncs', async function (t) { await s.close() }) -test('fsyncs is debounced', async function (t) { +test('fsync is debounced', async function (t) { const dir = await t.tmp() const s = new Storage(dir) @@ -33,15 +33,15 @@ test('fsyncs is debounced', async function (t) { const c = await s.createCore({ key: b4a.alloc(32, 3), discoveryKey: b4a.alloc(32, 3) }) const proms = [ - a.ensureFsynced(), - a.ensureFsynced(), - a.ensureFsynced(), - b.ensureFsynced(), - b.ensureFsynced(), - b.ensureFsynced(), - c.ensureFsynced(), - c.ensureFsynced(), - c.ensureFsynced() + a.fsync(), + a.fsync(), + a.fsync(), + b.fsync(), + b.fsync(), + b.fsync(), + c.fsync(), + c.fsync(), + c.fsync() ] await Promise.all(proms) @@ -49,7 +49,7 @@ test('fsyncs is debounced', async function (t) { // debounced t.is(s.fsyncs, 2) - await a.ensureFsynced() + await a.fsync() t.is(s.fsyncs, 3) From 421493746f07abdf53b4373cde7dc110b9782027 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Fri, 9 Oct 2026 15:59:00 +0100 Subject: [PATCH 10/13] fsyncsNeeded survives across session reopening --- index.js | 16 +++++++++++++++- test/fsync.js | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 98e91c9..3ffa38e 100644 --- a/index.js +++ b/index.js @@ -438,7 +438,16 @@ class HypercoreStorage { } fsyncsNeeded() { - return this.store.fsyncsStarted + 1 + const needed = this.store.needed.get(this.core.corePointer) + if (needed !== undefined) return needed + + return 1 // 1 guarantees fsync called at least once on store open + } + + markFsync() { + const needed = this.store.fsyncsStarted + 1 + this.store.needed.set(this.core.corePointer, needed) + return needed } fsync(target = this.store.fsyncsStarted + 1) { @@ -526,6 +535,7 @@ class CorestoreStorage { this.fsyncsStarted = 0 this.fsyncing = null this.lastFsyncAt = 0 + this.needed = new Map() if ((this.bootstrap && !this.readOnly && !this.allowBackup) || this.wait) { const corestoreFile = path.join(this.path, 'CORESTORE') @@ -605,6 +615,10 @@ class CorestoreStorage { this.lastFsyncAt = Date.now() this.fsyncs = gen + + for (const [corePointer, needed] of this.needed) { + if (needed <= gen) this.needed.delete(corePointer) + } } async audit() { diff --git a/test/fsync.js b/test/fsync.js index f962518..4c220ef 100644 --- a/test/fsync.js +++ b/test/fsync.js @@ -56,3 +56,51 @@ test('fsync is debounced', async function (t) { await a.close() await s.close() }) + +test('fsync needed is tracked per core and pruned on completion', async function (t) { + const s = await create(t) + await s.ready() + + const a = await s.createCore({ key: b4a.alloc(32, 1), discoveryKey: b4a.alloc(32, 1) }) + const b = await s.createCore({ key: b4a.alloc(32, 2), discoveryKey: b4a.alloc(32, 2) }) + + t.is(a.fsyncsNeeded(), 1) + t.is(b.fsyncsNeeded(), 1) + + t.is(a.markFsync(), 1) + t.is(a.fsyncsNeeded(), 1) + t.is(s.needed.size, 1) + + await s.flushFsync(1) + + t.is(s.fsyncs, 1) + t.is(s.needed.size, 0) + t.is(a.fsyncsNeeded(), 1) + + t.is(b.markFsync(), 2) + t.is(b.fsyncsNeeded(), 2) + + s.lastFsyncAt = 0 + const fsync = s.flushFsync(2) + + t.is(s.fsyncsStarted, 2) + t.is(a.markFsync(), 3) + + await fsync + + t.is(s.fsyncs, 2) + t.is(s.needed.size, 1) + t.is(a.fsyncsNeeded(), 3) + t.is(b.fsyncsNeeded(), 1) + + const reopened = await s.resumeCore(b4a.alloc(32, 1)) + t.is(reopened.fsyncsNeeded(), 3) + + await s.flushFsync(3) + t.is(s.needed.size, 0) + + await reopened.close() + await a.close() + await b.close() + await s.close() +}) From d94a887d292b31c952440f7c613d840041c1c67e Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Fri, 9 Oct 2026 16:06:25 +0100 Subject: [PATCH 11/13] flush when map size exceeds thres --- index.js | 6 ++++++ test/fsync.js | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/index.js b/index.js index 3ffa38e..e87ac84 100644 --- a/index.js +++ b/index.js @@ -11,6 +11,7 @@ const View = require('./lib/view.js') const VERSION = 2 const COLUMN_FAMILY = 'corestore' const MIN_FSYNC_GAP = 25 +const MAX_FSYNC_NEEDED = 4096 const { store, core } = require('./lib/keys.js') @@ -447,6 +448,11 @@ class HypercoreStorage { markFsync() { const needed = this.store.fsyncsStarted + 1 this.store.needed.set(this.core.corePointer, needed) + + if (this.store.needed.size >= MAX_FSYNC_NEEDED) { + this.store.flushFsync(needed).catch(noop) + } + return needed } diff --git a/test/fsync.js b/test/fsync.js index 4c220ef..a4adccb 100644 --- a/test/fsync.js +++ b/test/fsync.js @@ -104,3 +104,33 @@ test('fsync needed is tracked per core and pruned on completion', async function await b.close() await s.close() }) + +test('fsync is triggered when the needed map hits its size limit', async function (t) { + const s = await create(t) + await s.ready() + + const cores = [] + for (let i = 0; i < 4096; i++) { + const key = b4a.alloc(32) + key.writeUInt32BE(i) + cores.push(await s.createCore({ key, discoveryKey: key })) + } + + for (let i = 0; i < 4095; i++) cores[i].markFsync() + + t.is(s.needed.size, 4095) + t.is(s.fsyncing, null) + + cores[4095].markFsync() + + t.is(s.needed.size, 4096) + t.not(s.fsyncing, null) + + await s.fsyncing.promise + + t.is(s.fsyncs, 1) + t.is(s.needed.size, 0) + + for (const c of cores) await c.close() + await s.close() +}) From 98379e0a7d5fbd6bf8bec6c111c314604e434867 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Thu, 15 Oct 2026 16:29:23 +0100 Subject: [PATCH 12/13] encapsulate fsync tracking --- index.js | 73 +++++---------------------------------------------- test/fsync.js | 48 ++++++++++++++++----------------- 2 files changed, 30 insertions(+), 91 deletions(-) diff --git a/index.js b/index.js index e87ac84..b321a70 100644 --- a/index.js +++ b/index.js @@ -7,11 +7,10 @@ const fs = require('fs') const Xache = require('xache') const View = require('./lib/view.js') +const SyncTracker = require('./lib/sync-tracker.js') const VERSION = 2 const COLUMN_FAMILY = 'corestore' -const MIN_FSYNC_GAP = 25 -const MAX_FSYNC_NEEDED = 4096 const { store, core } = require('./lib/keys.js') @@ -439,25 +438,15 @@ class HypercoreStorage { } fsyncsNeeded() { - const needed = this.store.needed.get(this.core.corePointer) - if (needed !== undefined) return needed - - return 1 // 1 guarantees fsync called at least once on store open + return this.store.fsyncs.needed(this.core.corePointer) } markFsync() { - const needed = this.store.fsyncsStarted + 1 - this.store.needed.set(this.core.corePointer, needed) - - if (this.store.needed.size >= MAX_FSYNC_NEEDED) { - this.store.flushFsync(needed).catch(noop) - } - - return needed + return this.store.fsyncs.mark(this.core.corePointer) } - fsync(target = this.store.fsyncsStarted + 1) { - return this.store.flushFsync(target) + fsync(target) { + return this.store.fsyncs.flush(target) } close() { @@ -536,13 +525,6 @@ class CorestoreStorage { this.version = 0 this.migrating = null - // fsync state - this.fsyncs = 0 - this.fsyncsStarted = 0 - this.fsyncing = null - this.lastFsyncAt = 0 - this.needed = new Map() - if ((this.bootstrap && !this.readOnly && !this.allowBackup) || this.wait) { const corestoreFile = path.join(this.path, 'CORESTORE') @@ -565,6 +547,7 @@ class CorestoreStorage { } this.rocks = storage === null ? db : new RocksDB(dbPath, { ...opts, lock: this.deviceFile }) this.db = createColumnFamily(this.rocks, opts) + this.fsyncs = new SyncTracker(this.db) const { treeCache = { maxSize: 8192 } } = opts this.treeCache = new Xache(treeCache) @@ -587,46 +570,6 @@ class CorestoreStorage { return this.db.compactRange(opts) } - async flushFsync(target) { - while (this.fsyncs < target) { - if (this.db.closed) throw new Error('Storage has closed') - await this.fsync(target) - } - } - - async fsync(target) { - if (this.fsyncing !== null) { - if (target <= this.fsyncing.gen) return this.fsyncing.promise - await this.fsyncing.promise.catch(noop) - return - } - - const gen = this.fsyncsStarted + 1 - const promise = this._fsync(gen) - this.fsyncing = { gen, promise } - - try { - await promise - } finally { - this.fsyncing = null - } - } - - async _fsync(gen) { - const gap = MIN_FSYNC_GAP - (Date.now() - this.lastFsyncAt) - if (gap > 0) await sleep(gap) - - this.fsyncsStarted = gen - await this.db.flushWAL() - - this.lastFsyncAt = Date.now() - this.fsyncs = gen - - for (const [corePointer, needed] of this.needed) { - if (needed <= gen) this.needed.delete(corePointer) - } - } - async audit() { for await (const { core } of this.createCoreStream()) { const coreRx = new CoreRX(core, this.db, EMPTY, this.treeCache, this.stats, -1) @@ -1474,7 +1417,3 @@ async function getInfoFromBatch(db, c, discoveryKey, getAuth, getHead, getHints) hints: await hintsPromise } } - -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)) -} diff --git a/test/fsync.js b/test/fsync.js index a4adccb..8fe0654 100644 --- a/test/fsync.js +++ b/test/fsync.js @@ -9,14 +9,14 @@ test('fsync', async function (t) { const a = await s.createCore({ key: b4a.alloc(32, 1), discoveryKey: b4a.alloc(32, 1) }) - const target = a.store.fsyncsStarted + 1 + const target = s.fsyncs.started + 1 - t.is(s.fsyncs, 0) + t.is(s.fsyncs.completed, 0) t.is(target, 1) - await s.flushFsync(target) + await s.fsyncs.flush(target) - t.is(s.fsyncs, target) + t.is(s.fsyncs.completed, target) await a.close() await s.close() @@ -47,11 +47,11 @@ test('fsync is debounced', async function (t) { await Promise.all(proms) // debounced - t.is(s.fsyncs, 2) + t.is(s.fsyncs.completed, 2) await a.fsync() - t.is(s.fsyncs, 3) + t.is(s.fsyncs.completed, 3) await a.close() await s.close() @@ -69,35 +69,35 @@ test('fsync needed is tracked per core and pruned on completion', async function t.is(a.markFsync(), 1) t.is(a.fsyncsNeeded(), 1) - t.is(s.needed.size, 1) + t.is(s.fsyncs.pending.size, 1) - await s.flushFsync(1) + await s.fsyncs.flush(1) - t.is(s.fsyncs, 1) - t.is(s.needed.size, 0) + t.is(s.fsyncs.completed, 1) + t.is(s.fsyncs.pending.size, 0) t.is(a.fsyncsNeeded(), 1) t.is(b.markFsync(), 2) t.is(b.fsyncsNeeded(), 2) - s.lastFsyncAt = 0 - const fsync = s.flushFsync(2) + s.fsyncs.lastRunAt = 0 + const fsync = s.fsyncs.flush(2) - t.is(s.fsyncsStarted, 2) + t.is(s.fsyncs.started, 2) t.is(a.markFsync(), 3) await fsync - t.is(s.fsyncs, 2) - t.is(s.needed.size, 1) + t.is(s.fsyncs.completed, 2) + t.is(s.fsyncs.pending.size, 1) t.is(a.fsyncsNeeded(), 3) t.is(b.fsyncsNeeded(), 1) const reopened = await s.resumeCore(b4a.alloc(32, 1)) t.is(reopened.fsyncsNeeded(), 3) - await s.flushFsync(3) - t.is(s.needed.size, 0) + await s.fsyncs.flush(3) + t.is(s.fsyncs.pending.size, 0) await reopened.close() await a.close() @@ -118,18 +118,18 @@ test('fsync is triggered when the needed map hits its size limit', async functio for (let i = 0; i < 4095; i++) cores[i].markFsync() - t.is(s.needed.size, 4095) - t.is(s.fsyncing, null) + t.is(s.fsyncs.pending.size, 4095) + t.is(s.fsyncs.running, null) cores[4095].markFsync() - t.is(s.needed.size, 4096) - t.not(s.fsyncing, null) + t.is(s.fsyncs.pending.size, 4096) + t.not(s.fsyncs.running, null) - await s.fsyncing.promise + await s.fsyncs.running.promise - t.is(s.fsyncs, 1) - t.is(s.needed.size, 0) + t.is(s.fsyncs.completed, 1) + t.is(s.fsyncs.pending.size, 0) for (const c of cores) await c.close() await s.close() From 0ba3534205c2dabcf08437b584f95e3d9672f313 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Thu, 15 Oct 2026 16:31:52 +0100 Subject: [PATCH 13/13] add missing lib file --- index.js | 4 +-- lib/fsync-tracker.js | 75 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 lib/fsync-tracker.js diff --git a/index.js b/index.js index b321a70..383305c 100644 --- a/index.js +++ b/index.js @@ -7,7 +7,7 @@ const fs = require('fs') const Xache = require('xache') const View = require('./lib/view.js') -const SyncTracker = require('./lib/sync-tracker.js') +const FsyncTracker = require('./lib/fsync-tracker.js') const VERSION = 2 const COLUMN_FAMILY = 'corestore' @@ -547,7 +547,7 @@ class CorestoreStorage { } this.rocks = storage === null ? db : new RocksDB(dbPath, { ...opts, lock: this.deviceFile }) this.db = createColumnFamily(this.rocks, opts) - this.fsyncs = new SyncTracker(this.db) + this.fsyncs = new FsyncTracker(this.db) const { treeCache = { maxSize: 8192 } } = opts this.treeCache = new Xache(treeCache) diff --git a/lib/fsync-tracker.js b/lib/fsync-tracker.js new file mode 100644 index 0000000..5efc7a3 --- /dev/null +++ b/lib/fsync-tracker.js @@ -0,0 +1,75 @@ +const MIN_GAP = 25 +const MAX_PENDING = 4096 + +module.exports = class FsyncTracker { + constructor(db) { + this.db = db + this.completed = 0 + this.started = 0 + this.running = null + this.lastRunAt = 0 + this.pending = new Map() + } + + needed(ptr) { + const needed = this.pending.get(ptr) + if (needed !== undefined) return needed + + return 1 // 1 guarantees fsync called at least once on store open + } + + mark(ptr) { + const needed = this.started + 1 + this.pending.set(ptr, needed) + + if (this.pending.size >= MAX_PENDING) this.flush(needed).catch(noop) + + return needed + } + + async flush(target = this.started + 1) { + while (this.completed < target) { + if (this.db.closed) throw new Error('Storage has closed') + await this._sync(target) + } + } + + async _sync(target) { + if (this.running !== null) { + if (target <= this.running.gen) return this.running.promise + await this.running.promise.catch(noop) + return + } + + const gen = this.started + 1 + const promise = this._run(gen) + this.running = { gen, promise } + + try { + await promise + } finally { + this.running = null + } + } + + async _run(gen) { + const gap = MIN_GAP - (Date.now() - this.lastRunAt) + if (gap > 0) await sleep(gap) + + this.started = gen + await this.db.flushWAL() + + this.lastRunAt = Date.now() + this.completed = gen + + for (const [ptr, needed] of this.pending) { + if (needed <= gen) this.pending.delete(ptr) + } + } +} + +function noop() {} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)) +}