From 669fcb1a3efd5af83e86593362c6f5bbea4b257f Mon Sep 17 00:00:00 2001 From: Marcus Lagergren <1062473+lagergren@users.noreply.github.com> Date: Fri, 28 Aug 2026 14:28:15 +0200 Subject: [PATCH 1/2] Fix DirRepository scan-cache data race (blocks parallel compilation) (#547) --- .../main/java/org/xvm/asm/DirRepository.java | 22 +++- .../asm/DirRepositoryConcurrentScanTest.java | 113 ++++++++++++++++++ 2 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 javatools/src/test/java/org/xvm/asm/DirRepositoryConcurrentScanTest.java diff --git a/javatools/src/main/java/org/xvm/asm/DirRepository.java b/javatools/src/main/java/org/xvm/asm/DirRepository.java index 7922d9f352..b7a382f280 100644 --- a/javatools/src/main/java/org/xvm/asm/DirRepository.java +++ b/javatools/src/main/java/org/xvm/asm/DirRepository.java @@ -25,6 +25,22 @@ /** * A simple ModuleRepository that manages its contents in a directory. */ +/* + * THREAD SAFETY: the three public entry points below are synchronized on the repository. + * + * The scan cache is a plain HashMap (modulesByFile, a non-final field reassigned wholesale) and a + * plain TreeMap (modulesByName, rebuilt with clear() + a put() loop). Concurrent callers - two + * parallel compiles resolving library modules, say - otherwise race those rebuilds against get() + * and against the live keySet() view getModuleNames() hands out, which throws + * ConcurrentModificationException (and can spin or silently drop entries). + * + * A coarse per-repository lock is the right granularity here, not a pessimisation: this is a + * directory-scan cache consulted once per module lookup, not a hot inner loop. Note that swapping + * in concurrent maps, or copy-on-write with a volatile swap, would NOT be sufficient on its own, + * because ModuleInfo.ensureModule() ALSO lazily deserializes ("if (module == null) module = + * tryLoad()") - two threads would still duplicate that work and could publish a half-built module. + * The lock covers both the cache rebuild and the lazy materialization reached through it. + */ public class DirRepository implements ModuleRepository { // ----- constructors ------------------------------------------------------------------------- @@ -63,20 +79,20 @@ public boolean isReadOnly() { // ----- ModuleRepository API ------------------------------------------------------------------ @Override - public Set getModuleNames() { + public synchronized Set getModuleNames() { ensureCache(); return Collections.unmodifiableSet(modulesByName.keySet()); } @Override - public ModuleStructure loadModule(String sModule) { + public synchronized ModuleStructure loadModule(String sModule) { ensureCache(); ModuleInfo info = modulesByName.get(sModule); return info == null ? null : info.ensureModule(); } @Override - public void storeModule(ModuleStructure module) + public synchronized void storeModule(ModuleStructure module) throws IOException { if (m_fRO) { throw new IOException("repository is read-only: " + this); diff --git a/javatools/src/test/java/org/xvm/asm/DirRepositoryConcurrentScanTest.java b/javatools/src/test/java/org/xvm/asm/DirRepositoryConcurrentScanTest.java new file mode 100644 index 0000000000..cc83bdf190 --- /dev/null +++ b/javatools/src/test/java/org/xvm/asm/DirRepositoryConcurrentScanTest.java @@ -0,0 +1,113 @@ +package org.xvm.asm; + + +import java.io.File; + +import java.nio.file.Files; +import java.nio.file.Path; + +import java.util.ArrayList; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + + +/** + * Regression / red-on-master proof for the DirRepository scan-cache race. + * + *

{@code DirRepository} caches its directory scan in a plain {@code HashMap modulesByFile} (a + * non-final field reassigned by {@code rebuildCache}) and a plain {@code TreeMap modulesByName} + * (rebuilt with {@code clear()} + {@code put()}). On master none of {@code loadModule}/ + * {@code getModuleNames}/{@code ensureCache}/{@code rebuildCache} is synchronized, so concurrent + * callers race {@code clear()}/{@code put()} against {@code get()}/iteration. A fresh repository + * starts with {@code lastScan == 0}, so the first concurrent access has every thread run + * {@code rebuildCache} on the same maps at once. Two concurrent container-0 compiles reach exactly + * this path.

+ * + *

Proven red on master {@code 82683bcd2}: this exact test throws + * {@code java.util.ConcurrentModificationException} (verified in a master worktree with + * master-built modules parsed into the cache). It passes here because + * {@code DirRepository.loadModule}/{@code getModuleNames}/{@code storeModule} are {@code + * synchronized} on this branch (the coarse per-repository lock is the correct granularity for a + * lookup cache).

+ */ +public class DirRepositoryConcurrentScanTest { + private static final File LIB = + new File("xdk/build/install/xdk/lib").isDirectory() + ? new File("xdk/build/install/xdk/lib") + : new File("../xdk/build/install/xdk/lib"); + + @Test + public void concurrentScanDoesNotCorruptTheCache() throws Exception { + assumeTrue(LIB.isDirectory(), "need built .xtc modules to feed the repository"); + File[] xtc = LIB.listFiles((d, n) -> n.endsWith(".xtc")); + assumeTrue(xtc != null && xtc.length >= 4, "need several .xtc modules"); + + Path dir = Files.createTempDirectory("dirrepo-race"); + int copied = 0; + for (File f : xtc) { + if (f.getName().equals("ecstasy.xtc")) { + continue; // skip the big one to keep per-scan parse cost low + } + Files.copy(f.toPath(), dir.resolve(f.getName())); + if (++copied >= 6) { + break; + } + } + assumeTrue(copied >= 4, "need several small .xtc modules"); + + int threads = 8; + var pool = Executors.newFixedThreadPool(threads); + var failure = new AtomicReference(); + + try { + for (int iter = 0; iter < 80 && failure.get() == null; iter++) { + var repo = new DirRepository(dir.toFile(), true); + var start = new CountDownLatch(1); + var futures = new ArrayList>(); + for (int t = 0; t < threads; t++) { + futures.add(pool.submit(() -> { + start.await(); + // First concurrent call -> every thread runs rebuildCache (clear()+put()) + // on the shared modulesByName TreeMap; iterating the keySet view races it. + int seen = 0; + for (int i = 0; i < 8; i++) { + for (String name : repo.getModuleNames()) { + seen += name.length(); + } + } + return seen; + })); + } + start.countDown(); + for (Future f : futures) { + try { + f.get(20, TimeUnit.SECONDS); + } catch (ExecutionException e) { + failure.compareAndSet(null, e.getCause()); + } catch (TimeoutException e) { + failure.compareAndSet(null, new AssertionError( + "a scan thread hung (corrupted HashMap infinite loop?)", e)); + } + } + } + } finally { + pool.shutdownNow(); + } + + Throwable t = failure.get(); + if (t != null) { + fail("concurrent DirRepository scan corrupted the cache: " + t, t); + } + } +} From b035c8808183b40075bfc9e22e568f97403db144 Mon Sep 17 00:00:00 2001 From: Michael Raasch Date: Fri, 28 Aug 2026 14:11:19 +0000 Subject: [PATCH 2/2] Fix JsonMapStore Small->Medium transition resurrecting deleted keys When a JsonMapStore grows from the Small to the Medium storage model, both the commit() and retainTx() transition paths rewrite in-memory history values to the OffHeap marker. They did this unconditionally -- including for Deleted tombstones. Because OffHeap != Deleted, every key that had been removed became visible to keysAt() again, while sizeAt() (which tracks sizeByTx) still correctly excluded it. keysAt()'s `assert keys.size == size` then fails, and because the in-memory history stays corrupted the failure is sticky and cascades to every later read of that map. Guard both conversions with a `!= Deleted` check so tombstones are left intact. Adds a deterministic reproducer (JsonMapStoreTest.shouldKeepSizeAndKeysConsistentUnderConcurrentTransactions): model thresholds are injected low so the store crosses Small->Medium mid-run, then several Clients drive overlapping insert/delete/rollback transactions on one map while a reader checks size vs the iterated key count in a read transaction. Fails as `keys.size=520, size=412` before the fix; passes after. Full jsondb_test stays green (55/55). --- .../src/main/x/jsondb/storage/JsonMapStore.x | 15 +- .../jsondb_test/storage/JsonMapStoreTest.x | 132 ++++++++++++++++++ 2 files changed, 144 insertions(+), 3 deletions(-) diff --git a/lib_jsondb/src/main/x/jsondb/storage/JsonMapStore.x b/lib_jsondb/src/main/x/jsondb/storage/JsonMapStore.x index ce45548fb0..640bd2841d 100644 --- a/lib_jsondb/src/main/x/jsondb/storage/JsonMapStore.x +++ b/lib_jsondb/src/main/x/jsondb/storage/JsonMapStore.x @@ -871,7 +871,11 @@ service JsonMapStore // other entries in the history will be moved OffHeap during maintenance for ((Int txId, Array keys) : updatedKeysByTx) { for (Key key : keys) { - if (History valueHistory := history.get(key)) { + // A key deleted in this batch has a `Deleted` marker at `txId`; leave + // it be -- replacing it with `OffHeap` resurrects the removed key. + if (History valueHistory := history.get(key), + MapValue current := valueHistory.get(txId), + current != Deleted) { valueHistory.put(txId, OffHeap); } } @@ -943,10 +947,15 @@ service JsonMapStore if (modelAtCleanup != modelNow) { // the model has changed since the last clean-up if (modelAtCleanup <= Small && modelNow == Medium) { - // The model has grown to Medium, mark all history values as OffHeap + // The model has grown to Medium, mark all real history values as OffHeap. + // `Deleted` tombstones must be left intact -- overwriting one with `OffHeap` + // resurrects a removed key (it becomes visible to `keysAt()` again while + // `sizeAt()` still excludes it). for ((Key key, History valueHistory) : history) { for (Map.Entry entry : valueHistory.entries) { - entry.value = OffHeap; + if (entry.value != Deleted) { + entry.value = OffHeap; + } } } } else if (modelAtCleanup <= Medium && modelNow == Large) { diff --git a/manualTests/src/main/x/jsondb/jsondb_test/storage/JsonMapStoreTest.x b/manualTests/src/main/x/jsondb/jsondb_test/storage/JsonMapStoreTest.x index 5dd65c6b03..34874d2f07 100644 --- a/manualTests/src/main/x/jsondb/jsondb_test/storage/JsonMapStoreTest.x +++ b/manualTests/src/main/x/jsondb/jsondb_test/storage/JsonMapStoreTest.x @@ -521,4 +521,136 @@ class JsonMapStoreTest { } } } + + /** + * Regression test for a `keys.size == size` desync in `JsonMapStore` -- distinct from the + * (fixed) `keysAt()` merge-ordering bug above. + * + * When the store grew from the Small to the Medium model, both the `commit()` and the + * `retainTx()` transition paths rewrote history entries to the `OffHeap` marker + * *unconditionally* -- including `Deleted` tombstones. Since `OffHeap != Deleted`, every + * previously-removed key became visible to `keysAt()` again, while `sizeAt()` (which + * tracks `sizeByTx`) still correctly excluded it, so `keysAt()`'s `assert keys.size == + * size` failed and the store stayed desynced (`keysAt() > sizeAt()` by the number of + * resurrected tombstones). + * + * Seen in the field on the hottest DBMap as `"keys.size == size": keys.size=296, + * size=284` in an accumulated `--jsondb` run (durable log and `sizeByTx` agreed at 284; + * `history` reported 12 extra live entries -- resurrected deletes). + * + * Several `Client`s drive overlapping insert/delete/rollback transactions against one + * store (with the model thresholds injected low, so it crosses to Medium mid-run) while + * a reader repeatedly checks, inside a single read transaction, that `mapData.size` + * equals the iterated key count; `keysAt()`'s own internal assertion also surfaces here. + */ + @Test + @TestInjectables(Map:[JsonMapStore.ConfigSmallModelMaxFiles ="10", + JsonMapStore.ConfigMediumModelMaxFiles ="2000000", + JsonMapStore.ConfigSmallModelMaxBytes ="200000000", + JsonMapStore.ConfigMediumModelMaxBytes ="400000000"]) + void shouldKeepSizeAndKeysConsistentUnderConcurrentTransactions() { + assert TestClient boot := clientProvider.getClient(); + jsondb.Catalog catalog = boot.catalog; + TestSchema schema = boot.testSchema; + + // >10 distinct key-files => store transitions Small -> Medium, so values go + // off-heap and prepare()/keysAt() must loadValueFromOffHeap() (a real await). + Int seedCount = 40; + for (Int i : 0 ..< seedCount) { + schema.mapData.put($"seed-{i}", "s"); + } + assert schema.mapData.size == seedCount; + + Int writerCount = 6; + Int itersPerWriter = 30; + Int keysPerTx = 4; + + TxWorker[] writers = new TxWorker[writerCount] + (i -> new TxWorker(catalog.createClient().as(TestClient), i)); + TxWorker sampler = new TxWorker(catalog.createClient().as(TestClient), 99); + + @Future Int badSamples = sampler.checkParity(400); + @Volatile Int finished = 0; + @Future Tuple<> writersDone; + for (Int i : 0 ..< writerCount) { + @Future Int c = writers[i].runWrites(itersPerWriter, keysPerTx); + &c.whenComplete((n, e) -> { + if (++finished == writerCount) { &writersDone.set(()); } + }); + } + + Tuple<> joined = writersDone; + Int bad = badSamples; + + Int finalSize = schema.mapData.size; + Int finalKeys = schema.mapData.keys.toArray().size; + assert finalKeys == finalSize + as $"final size/keys desynced: size={finalSize} keys={finalKeys}"; + assert bad == 0 + as $"sizeAt()/keysAt() disagreed on {bad} of 400 in-flight reads"; + } + +/** + * Drives its own `Client` so several instances hit one `JsonMapStore` concurrently. + */ +static service TxWorker { + construct(TestClient client, Int id) { + this.client = client; + this.id = id; + } + + private TestClient client; + private Int id; + + /** + * `iterations` transactions: each inserts `keysPerTx` fresh keys and (from iteration 2 on) + * deletes 2 keys committed 2 iterations earlier. Every 3rd transaction rolls back. + * Returns the net committed key count. + */ + Int runWrites(Int iterations, Int keysPerTx) { + Connection conn = client.ensureConnection(); + TestSchema schema = client.testSchema; + Int committed = 0; + for (Int i : 0 ..< iterations) { + Boolean roll = (i % 3) == 2; + using (val tx = conn.createTransaction()) { + for (Int j : 0 ..< keysPerTx) { + schema.mapData.put($"w{id}-{i}-{j}", "v"); + } + if (i >= 2) { + schema.mapData.remove($"w{id}-{i-2}-0"); + schema.mapData.remove($"w{id}-{i-2}-1"); + } + if (roll) { + tx.rollbackOnly = True; + } + } + if (!roll) { + committed += (i >= 2 ? keysPerTx - 2 : keysPerTx); + } + } + return committed; + } + + /** + * `samples` times: inside one read transaction (single read view), compare `mapData.size` + * (→ `sizeAt`) with the iterated key count (→ `keysAt`). Returns the mismatch count; + * `keysAt()`'s internal `assert keys.size == size` throws straight out if tripped. + */ + Int checkParity(Int samples) { + Connection conn = client.ensureConnection(); + TestSchema schema = client.testSchema; + Int bad = 0; + for (Int i : 0 ..< samples) { + using (conn.createTransaction()) { + Int kc = schema.mapData.keys.toArray().size; + Int sz = schema.mapData.size; + if (kc != sz) { + ++bad; + } + } + } + return bad; + } +} }