Skip to content
18 changes: 18 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,15 @@ Check if a core exists.

List all cores. Stream data looks like this `{ discoveryKey, core }` where core contains the core header.

#### `await store.flushFsync(target)`

Wait until the WAL has been fsynced at least up to `target` (a value from `core.fsyncsNeeded()`).
Concurrent callers share the same fsync, which is paced to run at most every 25ms.

#### `store.fsyncs`

The latest fsync generation known to be on disk. Persisted and restored on open.

#### `await store.close()`

Close the storage instance.
Expand Down Expand Up @@ -203,6 +212,15 @@ Create a stream of all user data. `opts` is a query object with the following po
}
```

#### `target = core.fsyncsNeeded()`

Returns the fsync generation that will cover everything written so far. Store it with the head
(`fsyncsNeeded`) and pass it to `core.fsync(target)` once you need the writes to be durable.

#### `await core.fsync(target)`

Same as `store.flushFsync(target)`.

#### `await core.close()`

Close the core storage engine.
Expand Down
16 changes: 15 additions & 1 deletion index.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ const fs = require('fs')
const Xache = require('xache')

const View = require('./lib/view.js')
const FsyncTracker = require('./lib/fsync-tracker.js')

const VERSION = 2
const COLUMN_FAMILY = 'corestore'
Expand Down Expand Up @@ -436,6 +437,18 @@ class HypercoreStorage {
return new CoreTX(this.core, this.db, this.atom ? this.view : null, [])
}

fsyncsNeeded() {
return this.store.fsyncs.needed(this.core.corePointer)
}

markFsync() {
return this.store.fsyncs.mark(this.core.corePointer)
}

fsync(target) {
return this.store.fsyncs.flush(target)
}

close() {
if (this.view !== null) {
this.view.readStop()
Expand Down Expand Up @@ -534,6 +547,7 @@ class CorestoreStorage {
}
this.rocks = storage === null ? db : new RocksDB(dbPath, { ...opts, lock: this.deviceFile })
this.db = createColumnFamily(this.rocks, opts)
this.fsyncs = new FsyncTracker(this.db)

const { treeCache = { maxSize: 8192 } } = opts
this.treeCache = new Xache(treeCache)
Expand All @@ -549,7 +563,7 @@ class CorestoreStorage {

async ready() {
if (this.version === 0) await this._migrateStore()
return this.db.ready()
await this.db.ready()
}

compact(opts) {
Expand Down
75 changes: 75 additions & 0 deletions lib/fsync-tracker.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
const MIN_GAP = 25
const MAX_PENDING = 4096

module.exports = class FsyncTracker {
constructor(db) {
this.db = db
this.completed = 0
this.started = 0
this.running = null
this.lastRunAt = 0
this.pending = new Map()
}

needed(ptr) {
const needed = this.pending.get(ptr)
if (needed !== undefined) return needed

return 1 // 1 guarantees fsync called at least once on store open
}

mark(ptr) {
const needed = this.started + 1
this.pending.set(ptr, needed)

if (this.pending.size >= MAX_PENDING) this.flush(needed).catch(noop)

return needed
}

async flush(target = this.started + 1) {
while (this.completed < target) {
if (this.db.closed) throw new Error('Storage has closed')
await this._sync(target)
}
}

async _sync(target) {
if (this.running !== null) {
if (target <= this.running.gen) return this.running.promise
await this.running.promise.catch(noop)
return
}

const gen = this.started + 1
const promise = this._run(gen)
this.running = { gen, promise }

try {
await promise
} finally {
this.running = null
}
}

async _run(gen) {
const gap = MIN_GAP - (Date.now() - this.lastRunAt)
if (gap > 0) await sleep(gap)

this.started = gen
await this.db.flushWAL()

this.lastRunAt = Date.now()
this.completed = gen

for (const [ptr, needed] of this.pending) {
if (needed <= gen) this.pending.delete(ptr)
}
}
}

function noop() {}

function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms))
}
1 change: 1 addition & 0 deletions test/all.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ async function runTests() {
await import('./atomic.js')
await import('./basic.js')
await import('./core.js')
await import('./fsync.js')
await import('./groups.js')
await import('./snapshot.js')
await import('./streams.js')
Expand Down
8 changes: 7 additions & 1 deletion test/basic.js
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,13 @@ test('can get info from store efficiently', async function (t) {
keyPair: null,
encryptionKey: null
},
head: { fork: 0, length: 2, rootHash: b4a.alloc(32, 0), signature: null, timestamp: 0 },
head: {
fork: 0,
length: 2,
rootHash: b4a.alloc(32, 0),
signature: null,
timestamp: 0
},
hints: null
})
}
Expand Down
136 changes: 136 additions & 0 deletions test/fsync.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
const test = require('brittle')
const b4a = require('b4a')
const Storage = require('../')
const { create } = require('./helpers')

test('fsync', async function (t) {
const s = await create(t)
await s.ready()

const a = await s.createCore({ key: b4a.alloc(32, 1), discoveryKey: b4a.alloc(32, 1) })

const target = s.fsyncs.started + 1

t.is(s.fsyncs.completed, 0)
t.is(target, 1)

await s.fsyncs.flush(target)

t.is(s.fsyncs.completed, target)

await a.close()
await s.close()
})

test('fsync is debounced', async function (t) {
const dir = await t.tmp()

const s = new Storage(dir)
await s.ready()

const a = await s.createCore({ key: b4a.alloc(32, 1), discoveryKey: b4a.alloc(32, 1) })
const b = await s.createCore({ key: b4a.alloc(32, 2), discoveryKey: b4a.alloc(32, 2) })
const c = await s.createCore({ key: b4a.alloc(32, 3), discoveryKey: b4a.alloc(32, 3) })

const proms = [
a.fsync(),
a.fsync(),
a.fsync(),
b.fsync(),
b.fsync(),
b.fsync(),
c.fsync(),
c.fsync(),
c.fsync()
]

await Promise.all(proms)

// debounced
t.is(s.fsyncs.completed, 2)

await a.fsync()

t.is(s.fsyncs.completed, 3)

await a.close()
await s.close()
})

test('fsync needed is tracked per core and pruned on completion', async function (t) {
const s = await create(t)
await s.ready()

const a = await s.createCore({ key: b4a.alloc(32, 1), discoveryKey: b4a.alloc(32, 1) })
const b = await s.createCore({ key: b4a.alloc(32, 2), discoveryKey: b4a.alloc(32, 2) })

t.is(a.fsyncsNeeded(), 1)
t.is(b.fsyncsNeeded(), 1)

t.is(a.markFsync(), 1)
t.is(a.fsyncsNeeded(), 1)
t.is(s.fsyncs.pending.size, 1)

await s.fsyncs.flush(1)

t.is(s.fsyncs.completed, 1)
t.is(s.fsyncs.pending.size, 0)
t.is(a.fsyncsNeeded(), 1)

t.is(b.markFsync(), 2)
t.is(b.fsyncsNeeded(), 2)

s.fsyncs.lastRunAt = 0
const fsync = s.fsyncs.flush(2)

t.is(s.fsyncs.started, 2)
t.is(a.markFsync(), 3)

await fsync

t.is(s.fsyncs.completed, 2)
t.is(s.fsyncs.pending.size, 1)
t.is(a.fsyncsNeeded(), 3)
t.is(b.fsyncsNeeded(), 1)

const reopened = await s.resumeCore(b4a.alloc(32, 1))
t.is(reopened.fsyncsNeeded(), 3)

await s.fsyncs.flush(3)
t.is(s.fsyncs.pending.size, 0)

await reopened.close()
await a.close()
await b.close()
await s.close()
})

test('fsync is triggered when the needed map hits its size limit', async function (t) {
const s = await create(t)
await s.ready()

const cores = []
for (let i = 0; i < 4096; i++) {
const key = b4a.alloc(32)
key.writeUInt32BE(i)
cores.push(await s.createCore({ key, discoveryKey: key }))
}

for (let i = 0; i < 4095; i++) cores[i].markFsync()

t.is(s.fsyncs.pending.size, 4095)
t.is(s.fsyncs.running, null)

cores[4095].markFsync()

t.is(s.fsyncs.pending.size, 4096)
t.not(s.fsyncs.running, null)

await s.fsyncs.running.promise

t.is(s.fsyncs.completed, 1)
t.is(s.fsyncs.pending.size, 0)

for (const c of cores) await c.close()
await s.close()
})
Loading