Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 19 additions & 3 deletions javatools/src/main/java/org/xvm/asm/DirRepository.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,22 @@
/**
* A simple ModuleRepository that manages its contents in a directory.
*/
/*
* THREAD SAFETY: the three public entry points below are synchronized on the repository.
*
* The scan cache is a plain HashMap (modulesByFile, a non-final field reassigned wholesale) and a
* plain TreeMap (modulesByName, rebuilt with clear() + a put() loop). Concurrent callers - two
* parallel compiles resolving library modules, say - otherwise race those rebuilds against get()
* and against the live keySet() view getModuleNames() hands out, which throws
* ConcurrentModificationException (and can spin or silently drop entries).
*
* A coarse per-repository lock is the right granularity here, not a pessimisation: this is a
* directory-scan cache consulted once per module lookup, not a hot inner loop. Note that swapping
* in concurrent maps, or copy-on-write with a volatile swap, would NOT be sufficient on its own,
* because ModuleInfo.ensureModule() ALSO lazily deserializes ("if (module == null) module =
* tryLoad()") - two threads would still duplicate that work and could publish a half-built module.
* The lock covers both the cache rebuild and the lazy materialization reached through it.
*/
public class DirRepository
implements ModuleRepository {
// ----- constructors -------------------------------------------------------------------------
Expand Down Expand Up @@ -63,20 +79,20 @@ public boolean isReadOnly() {
// ----- ModuleRepository API ------------------------------------------------------------------

@Override
public Set<String> getModuleNames() {
public synchronized Set<String> getModuleNames() {
ensureCache();
return Collections.unmodifiableSet(modulesByName.keySet());
}

@Override
public ModuleStructure loadModule(String sModule) {
public synchronized ModuleStructure loadModule(String sModule) {
ensureCache();
ModuleInfo info = modulesByName.get(sModule);
return info == null ? null : info.ensureModule();
}

@Override
public void storeModule(ModuleStructure module)
public synchronized void storeModule(ModuleStructure module)
throws IOException {
if (m_fRO) {
throw new IOException("repository is read-only: " + this);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
package org.xvm.asm;


import java.io.File;

import java.nio.file.Files;
import java.nio.file.Path;

import java.util.ArrayList;

import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;

import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.fail;
import static org.junit.jupiter.api.Assumptions.assumeTrue;


/**
* Regression / red-on-master proof for the DirRepository scan-cache race.
*
* <p>{@code DirRepository} caches its directory scan in a plain {@code HashMap modulesByFile} (a
* non-final field reassigned by {@code rebuildCache}) and a plain {@code TreeMap modulesByName}
* (rebuilt with {@code clear()} + {@code put()}). On master none of {@code loadModule}/
* {@code getModuleNames}/{@code ensureCache}/{@code rebuildCache} is synchronized, so concurrent
* callers race {@code clear()}/{@code put()} against {@code get()}/iteration. A fresh repository
* starts with {@code lastScan == 0}, so the first concurrent access has every thread run
* {@code rebuildCache} on the same maps at once. Two concurrent container-0 compiles reach exactly
* this path.</p>
*
* <p><b>Proven red on master</b> {@code 82683bcd2}: this exact test throws
* {@code java.util.ConcurrentModificationException} (verified in a master worktree with
* master-built modules parsed into the cache). It passes here because
* {@code DirRepository.loadModule}/{@code getModuleNames}/{@code storeModule} are {@code
* synchronized} on this branch (the coarse per-repository lock is the correct granularity for a
* lookup cache).</p>
*/
public class DirRepositoryConcurrentScanTest {
private static final File LIB =
new File("xdk/build/install/xdk/lib").isDirectory()
? new File("xdk/build/install/xdk/lib")
: new File("../xdk/build/install/xdk/lib");

@Test
public void concurrentScanDoesNotCorruptTheCache() throws Exception {
assumeTrue(LIB.isDirectory(), "need built .xtc modules to feed the repository");
File[] xtc = LIB.listFiles((d, n) -> n.endsWith(".xtc"));
assumeTrue(xtc != null && xtc.length >= 4, "need several .xtc modules");

Path dir = Files.createTempDirectory("dirrepo-race");
int copied = 0;
for (File f : xtc) {
if (f.getName().equals("ecstasy.xtc")) {
continue; // skip the big one to keep per-scan parse cost low
}
Files.copy(f.toPath(), dir.resolve(f.getName()));
if (++copied >= 6) {
break;
}
}
assumeTrue(copied >= 4, "need several small .xtc modules");

int threads = 8;
var pool = Executors.newFixedThreadPool(threads);
var failure = new AtomicReference<Throwable>();

try {
for (int iter = 0; iter < 80 && failure.get() == null; iter++) {
var repo = new DirRepository(dir.toFile(), true);
var start = new CountDownLatch(1);
var futures = new ArrayList<Future<?>>();
for (int t = 0; t < threads; t++) {
futures.add(pool.submit(() -> {
start.await();
// First concurrent call -> every thread runs rebuildCache (clear()+put())
// on the shared modulesByName TreeMap; iterating the keySet view races it.
int seen = 0;
for (int i = 0; i < 8; i++) {
for (String name : repo.getModuleNames()) {
seen += name.length();
}
}
return seen;
}));
}
start.countDown();
for (Future<?> f : futures) {
try {
f.get(20, TimeUnit.SECONDS);
} catch (ExecutionException e) {
failure.compareAndSet(null, e.getCause());
} catch (TimeoutException e) {
failure.compareAndSet(null, new AssertionError(
"a scan thread hung (corrupted HashMap infinite loop?)", e));
}
}
}
} finally {
pool.shutdownNow();
}

Throwable t = failure.get();
if (t != null) {
fail("concurrent DirRepository scan corrupted the cache: " + t, t);
}
}
}
15 changes: 12 additions & 3 deletions lib_jsondb/src/main/x/jsondb/storage/JsonMapStore.x
Original file line number Diff line number Diff line change
Expand Up @@ -871,7 +871,11 @@ service JsonMapStore<Key extends immutable Const, Value extends immutable Const>
// other entries in the history will be moved OffHeap during maintenance
for ((Int txId, Array<Key> keys) : updatedKeysByTx) {
for (Key key : keys) {
if (History valueHistory := history.get(key)) {
// A key deleted in this batch has a `Deleted` marker at `txId`; leave
// it be -- replacing it with `OffHeap` resurrects the removed key.
if (History valueHistory := history.get(key),
MapValue current := valueHistory.get(txId),
current != Deleted) {
valueHistory.put(txId, OffHeap);
}
}
Expand Down Expand Up @@ -943,10 +947,15 @@ service JsonMapStore<Key extends immutable Const, Value extends immutable Const>
if (modelAtCleanup != modelNow) {
// the model has changed since the last clean-up
if (modelAtCleanup <= Small && modelNow == Medium) {
// The model has grown to Medium, mark all history values as OffHeap
// The model has grown to Medium, mark all real history values as OffHeap.
// `Deleted` tombstones must be left intact -- overwriting one with `OffHeap`
// resurrects a removed key (it becomes visible to `keysAt()` again while
// `sizeAt()` still excludes it).
for ((Key key, History valueHistory) : history) {
for (Map.Entry<Int, MapValue> entry : valueHistory.entries) {
entry.value = OffHeap;
if (entry.value != Deleted) {
entry.value = OffHeap;
}
}
}
} else if (modelAtCleanup <= Medium && modelNow == Large) {
Expand Down
132 changes: 132 additions & 0 deletions manualTests/src/main/x/jsondb/jsondb_test/storage/JsonMapStoreTest.x
Original file line number Diff line number Diff line change
Expand Up @@ -521,4 +521,136 @@ class JsonMapStoreTest {
}
}
}

/**
* Regression test for a `keys.size == size` desync in `JsonMapStore` -- distinct from the
* (fixed) `keysAt()` merge-ordering bug above.
*
* When the store grew from the Small to the Medium model, both the `commit()` and the
* `retainTx()` transition paths rewrote history entries to the `OffHeap` marker
* *unconditionally* -- including `Deleted` tombstones. Since `OffHeap != Deleted`, every
* previously-removed key became visible to `keysAt()` again, while `sizeAt()` (which
* tracks `sizeByTx`) still correctly excluded it, so `keysAt()`'s `assert keys.size ==
* size` failed and the store stayed desynced (`keysAt() > sizeAt()` by the number of
* resurrected tombstones).
*
* Seen in the field on the hottest DBMap as `"keys.size == size": keys.size=296,
* size=284` in an accumulated `--jsondb` run (durable log and `sizeByTx` agreed at 284;
* `history` reported 12 extra live entries -- resurrected deletes).
*
* Several `Client`s drive overlapping insert/delete/rollback transactions against one
* store (with the model thresholds injected low, so it crosses to Medium mid-run) while
* a reader repeatedly checks, inside a single read transaction, that `mapData.size`
* equals the iterated key count; `keysAt()`'s own internal assertion also surfaces here.
*/
@Test
@TestInjectables(Map:[JsonMapStore.ConfigSmallModelMaxFiles ="10",
JsonMapStore.ConfigMediumModelMaxFiles ="2000000",
JsonMapStore.ConfigSmallModelMaxBytes ="200000000",
JsonMapStore.ConfigMediumModelMaxBytes ="400000000"])
void shouldKeepSizeAndKeysConsistentUnderConcurrentTransactions() {
assert TestClient boot := clientProvider.getClient();
jsondb.Catalog<TestSchema> catalog = boot.catalog;
TestSchema schema = boot.testSchema;

// >10 distinct key-files => store transitions Small -> Medium, so values go
// off-heap and prepare()/keysAt() must loadValueFromOffHeap() (a real await).
Int seedCount = 40;
for (Int i : 0 ..< seedCount) {
schema.mapData.put($"seed-{i}", "s");
}
assert schema.mapData.size == seedCount;

Int writerCount = 6;
Int itersPerWriter = 30;
Int keysPerTx = 4;

TxWorker[] writers = new TxWorker[writerCount]
(i -> new TxWorker(catalog.createClient().as(TestClient), i));
TxWorker sampler = new TxWorker(catalog.createClient().as(TestClient), 99);

@Future Int badSamples = sampler.checkParity(400);
@Volatile Int finished = 0;
@Future Tuple<> writersDone;
for (Int i : 0 ..< writerCount) {
@Future Int c = writers[i].runWrites(itersPerWriter, keysPerTx);
&c.whenComplete((n, e) -> {
if (++finished == writerCount) { &writersDone.set(()); }
});
}

Tuple<> joined = writersDone;
Int bad = badSamples;

Int finalSize = schema.mapData.size;
Int finalKeys = schema.mapData.keys.toArray().size;
assert finalKeys == finalSize
as $"final size/keys desynced: size={finalSize} keys={finalKeys}";
assert bad == 0
as $"sizeAt()/keysAt() disagreed on {bad} of 400 in-flight reads";
}

/**
* Drives its own `Client` so several instances hit one `JsonMapStore` concurrently.
*/
static service TxWorker {
construct(TestClient client, Int id) {
this.client = client;
this.id = id;
}

private TestClient client;
private Int id;

/**
* `iterations` transactions: each inserts `keysPerTx` fresh keys and (from iteration 2 on)
* deletes 2 keys committed 2 iterations earlier. Every 3rd transaction rolls back.
* Returns the net committed key count.
*/
Int runWrites(Int iterations, Int keysPerTx) {
Connection<TestSchema> conn = client.ensureConnection();
TestSchema schema = client.testSchema;
Int committed = 0;
for (Int i : 0 ..< iterations) {
Boolean roll = (i % 3) == 2;
using (val tx = conn.createTransaction()) {
for (Int j : 0 ..< keysPerTx) {
schema.mapData.put($"w{id}-{i}-{j}", "v");
}
if (i >= 2) {
schema.mapData.remove($"w{id}-{i-2}-0");
schema.mapData.remove($"w{id}-{i-2}-1");
}
if (roll) {
tx.rollbackOnly = True;
}
}
if (!roll) {
committed += (i >= 2 ? keysPerTx - 2 : keysPerTx);
}
}
return committed;
}

/**
* `samples` times: inside one read transaction (single read view), compare `mapData.size`
* (→ `sizeAt`) with the iterated key count (→ `keysAt`). Returns the mismatch count;
* `keysAt()`'s internal `assert keys.size == size` throws straight out if tripped.
*/
Int checkParity(Int samples) {
Connection<TestSchema> conn = client.ensureConnection();
TestSchema schema = client.testSchema;
Int bad = 0;
for (Int i : 0 ..< samples) {
using (conn.createTransaction()) {
Int kc = schema.mapData.keys.toArray().size;
Int sz = schema.mapData.size;
if (kc != sz) {
++bad;
}
}
}
return bad;
}
}
}
Loading