From cccc493fe2badde8784000e8d344d5dcb7b96be1 Mon Sep 17 00:00:00 2001 From: hdegroote <75906619+HDegroote@users.noreply.github.com> Date: Tue, 10 Mar 2026 13:04:09 +0100 Subject: [PATCH 1/5] Add test to illustrate data loss --- test/basic.js | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/test/basic.js b/test/basic.js index ec06963..0d402c2 100644 --- a/test/basic.js +++ b/test/basic.js @@ -1,3 +1,4 @@ +const ScopeLock = require('scope-lock') const definition = require('./fixtures/definition') const { test } = require('./helpers') const tmp = require('test-tmp') @@ -607,3 +608,63 @@ test('enum as key type', async function ({ create, bee }, t) { await db.close() }) + +test.solo('concurrency does not cause missed data', async function ({ create }, t) { + t.timeout(300_000) + const db = await create(definition) + + const entries = 100_000 + const batchSize = 10_000 + const flushBatchInterval = 50 + const iterations = 5 + + console.log('expected entries per iteration:', entries) + console.log('expected totl entries:', entries * iterations) + const lock = new ScopeLock({ debounce: true }) + const interval = setInterval(async () => { + // Debounce the flush + if (!(await lock.lock())) return + try { + if (db.updates.size > 0) { + console.log(`flushing ${db.updates.size} updates`) + await db.flush() + } + } finally { + lock.unlock() + } + }, flushBatchInterval) + + for (let i = 0; i < iterations; i++) { + let proms = [] + console.log('Adding', entries, 'entries...') + for (let i = 1; i < entries + 1; i++) { + proms.push(db.insert('members', { id: `${Math.random()}`, age: 25 })) + if (i % batchSize === 0) { + await Promise.all(proms) + proms = [] + } + } + + await Promise.all(proms) + console.log('added', entries, 'entries...') + + let nrEntries = 0 + for await (const e of db.find('members')) nrEntries++ + console.log('iteration', i, 'nr entries try 1', nrEntries) + + nrEntries = 0 + for await (const e of db.find('members')) nrEntries++ + console.log('iteration', i, 'nr entries try 2', nrEntries, ' entries') + } + + // some extra time to ensure it's all flushed + await new Promise((resolve) => setTimeout(resolve, 5000)) + + let nrEntries = 0 + for await (const e of db.find('members')) nrEntries++ + console.log('final entries:', nrEntries) + t.is(nrEntries, entries * iterations, 'Added all entries') + + clearInterval(interval) + await db.close() +}) From 9e41d3a86b853ff09e0792256ff1c6b7611de004 Mon Sep 17 00:00:00 2001 From: hdegroote <75906619+HDegroote@users.noreply.github.com> Date: Tue, 10 Mar 2026 13:09:26 +0100 Subject: [PATCH 2/5] lint --- test/basic.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/basic.js b/test/basic.js index 0d402c2..9c71a34 100644 --- a/test/basic.js +++ b/test/basic.js @@ -649,11 +649,11 @@ test.solo('concurrency does not cause missed data', async function ({ create }, console.log('added', entries, 'entries...') let nrEntries = 0 - for await (const e of db.find('members')) nrEntries++ + for await (const _ of db.find('members')) nrEntries++ console.log('iteration', i, 'nr entries try 1', nrEntries) nrEntries = 0 - for await (const e of db.find('members')) nrEntries++ + for await (const _ of db.find('members')) nrEntries++ console.log('iteration', i, 'nr entries try 2', nrEntries, ' entries') } @@ -661,7 +661,7 @@ test.solo('concurrency does not cause missed data', async function ({ create }, await new Promise((resolve) => setTimeout(resolve, 5000)) let nrEntries = 0 - for await (const e of db.find('members')) nrEntries++ + for await (const _ of db.find('members')) nrEntries++ console.log('final entries:', nrEntries) t.is(nrEntries, entries * iterations, 'Added all entries') From 70052e58ed01e6dbabf414c32419564d8589cbad Mon Sep 17 00:00:00 2001 From: hdegroote <75906619+HDegroote@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:38:09 +0100 Subject: [PATCH 3/5] Fix loop variable + await each insert --- test/basic.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/basic.js b/test/basic.js index 9c71a34..0d3f4b7 100644 --- a/test/basic.js +++ b/test/basic.js @@ -637,12 +637,13 @@ test.solo('concurrency does not cause missed data', async function ({ create }, for (let i = 0; i < iterations; i++) { let proms = [] console.log('Adding', entries, 'entries...') - for (let i = 1; i < entries + 1; i++) { - proms.push(db.insert('members', { id: `${Math.random()}`, age: 25 })) - if (i % batchSize === 0) { - await Promise.all(proms) - proms = [] - } + for (let j = 1; j < entries + 1; j++) { + await db.insert('members', { id: `${i + j * 100_000}`, age: 25 }) + // proms.push(db.insert('members', { id: `${i + j * 100_000}`, age: 25 })) + // if (i % batchSize === 0) { + // await Promise.all(proms) + // proms = [] + // } } await Promise.all(proms) From f38863f06dda981665524f4df0721e4f5a00c3cd Mon Sep 17 00:00:00 2001 From: hdegroote <75906619+HDegroote@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:59:02 +0100 Subject: [PATCH 4/5] Fix setup + use transaction --- test/basic.js | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/test/basic.js b/test/basic.js index 0d3f4b7..db683a4 100644 --- a/test/basic.js +++ b/test/basic.js @@ -620,14 +620,17 @@ test.solo('concurrency does not cause missed data', async function ({ create }, console.log('expected entries per iteration:', entries) console.log('expected totl entries:', entries * iterations) + let tx = db.transaction() const lock = new ScopeLock({ debounce: true }) const interval = setInterval(async () => { - // Debounce the flush - if (!(await lock.lock())) return + // Debounce the flush + if (!(await lock.lock())) return try { - if (db.updates.size > 0) { - console.log(`flushing ${db.updates.size} updates`) - await db.flush() + if (tx.updates.size > 0) { + const prom = tx.flush() + console.log(`flushing ${tx.updates.size} updates`) + tx = db.transaction() + await prom } } finally { lock.unlock() @@ -638,12 +641,12 @@ test.solo('concurrency does not cause missed data', async function ({ create }, let proms = [] console.log('Adding', entries, 'entries...') for (let j = 1; j < entries + 1; j++) { - await db.insert('members', { id: `${i + j * 100_000}`, age: 25 }) - // proms.push(db.insert('members', { id: `${i + j * 100_000}`, age: 25 })) - // if (i % batchSize === 0) { - // await Promise.all(proms) - // proms = [] - // } + // await tx.insert('members', { id: `${i * 1_000_000 + j}`, age: 25 }) + proms.push(tx.insert('members', { id: `${i * 1_000_000 + j }`, age: 25 })) + if (j % batchSize === 0) { + console.log(`Awaiting promise batch: ${proms.length}`) + await Promise.all(proms) + } } await Promise.all(proms) From e08f0b54c47c86ace338661742475b5ec4da1859 Mon Sep 17 00:00:00 2001 From: hdegroote <75906619+HDegroote@users.noreply.github.com> Date: Thu, 12 Mar 2026 22:18:59 +0100 Subject: [PATCH 5/5] Simplify failing test --- test/basic.js | 58 ++++++--------------------------------------------- 1 file changed, 6 insertions(+), 52 deletions(-) diff --git a/test/basic.js b/test/basic.js index db683a4..68965ae 100644 --- a/test/basic.js +++ b/test/basic.js @@ -610,65 +610,19 @@ test('enum as key type', async function ({ create, bee }, t) { }) test.solo('concurrency does not cause missed data', async function ({ create }, t) { - t.timeout(300_000) const db = await create(definition) - const entries = 100_000 - const batchSize = 10_000 - const flushBatchInterval = 50 - const iterations = 5 - - console.log('expected entries per iteration:', entries) - console.log('expected totl entries:', entries * iterations) - let tx = db.transaction() - const lock = new ScopeLock({ debounce: true }) - const interval = setInterval(async () => { - // Debounce the flush - if (!(await lock.lock())) return - try { - if (tx.updates.size > 0) { - const prom = tx.flush() - console.log(`flushing ${tx.updates.size} updates`) - tx = db.transaction() - await prom - } - } finally { - lock.unlock() - } - }, flushBatchInterval) - - for (let i = 0; i < iterations; i++) { - let proms = [] - console.log('Adding', entries, 'entries...') - for (let j = 1; j < entries + 1; j++) { - // await tx.insert('members', { id: `${i * 1_000_000 + j}`, age: 25 }) - proms.push(tx.insert('members', { id: `${i * 1_000_000 + j }`, age: 25 })) - if (j % batchSize === 0) { - console.log(`Awaiting promise batch: ${proms.length}`) - await Promise.all(proms) - } - } + const tx = db.transaction() + await tx.insert('members', { id: '1', age: 25 }) - await Promise.all(proms) - console.log('added', entries, 'entries...') - - let nrEntries = 0 - for await (const _ of db.find('members')) nrEntries++ - console.log('iteration', i, 'nr entries try 1', nrEntries) - - nrEntries = 0 - for await (const _ of db.find('members')) nrEntries++ - console.log('iteration', i, 'nr entries try 2', nrEntries, ' entries') - } - - // some extra time to ensure it's all flushed - await new Promise((resolve) => setTimeout(resolve, 5000)) + const prom = tx.insert('members', { id: '2', age: 25 }) + await Promise.all([tx.flush(), prom]) let nrEntries = 0 for await (const _ of db.find('members')) nrEntries++ console.log('final entries:', nrEntries) - t.is(nrEntries, entries * iterations, 'Added all entries') - clearInterval(interval) + t.is(nrEntries, 2, 'unfinished insert is not lost') + await db.close() })