diff --git a/cmd/stellar-rpc/internal/rpcv2/stores/event/concurrent_bitmaps.go b/cmd/stellar-rpc/internal/rpcv2/stores/event/concurrent_bitmaps.go index dc6c44d08..ad2d2cefd 100644 --- a/cmd/stellar-rpc/internal/rpcv2/stores/event/concurrent_bitmaps.go +++ b/cmd/stellar-rpc/internal/rpcv2/stores/event/concurrent_bitmaps.go @@ -7,117 +7,101 @@ import ( "github.com/RoaringBitmap/roaring/v2" ) -// promotionThreshold is the number of event IDs stored in a list -// before promoting to a roaring bitmap. Most terms are sparse and -// a list is more memory-efficient than a roaring bitmap for small sets. -// -// Value of 64 chosen to comfortably exceed the observed mean -// cardinality (~14.5–16.3 events per term across production chunks -// 005901–005908; see BenchmarkEventIndex_10M for the modeled -// distribution). Terms below the threshold stay in list mode -// (≈256 B per slice); only long-tail dense terms promote to roaring. +// promotionThreshold is the number of event IDs a term holds in a +// sorted list before it is promoted to a roaring bitmap. Observed +// mean cardinality is ~14.5–16.3 events per term (production chunks +// 005901–005908; see BenchmarkEventIndex_10M), so most terms stay +// in list mode. const promotionThreshold = 64 -// termState is an immutable snapshot of a single term's event IDs. -// At most one of ids / bm is non-nil at any moment: -// - Sparse mode: ids holds the sorted []uint32 of event IDs. -// - Dense mode: bm holds the roaring.Bitmap (with COW enabled so -// subsequent Clones in AddTo are O(1) shallow copies). +// denseState is the state of one dense term. One is allocated at +// promotion and lives for the term's lifetime. // -// A new termState is allocated and atomically published on every -// AddTo. Readers atomic.Load the pointer and operate on the -// resulting struct — by construction the (ids, bm) pair is always -// observed consistently, so the previous (bm.Load, ids.Load) split -// and its promotion-window re-Load workaround are gone. +// - wbm is the writer's private bitmap. AddTo mutates it in place. +// It is never handed to a reader. +// - pub is the last immutable snapshot handed to readers. nil means +// wbm holds writes that no snapshot has: nil at promotion, nil +// again after every write, non-nil only once a reader has cloned +// wbm since the last write. +// - mu serializes AddMany on wbm against Clone of wbm. Both write +// wbm's copy-on-write flags. Every store to pub happens under mu. +type denseState struct { + mu sync.Mutex + wbm *roaring.Bitmap + pub atomic.Pointer[roaring.Bitmap] +} + +// termState is the immutable per-term entry. Exactly one field is set. +// - Sparse: ids is the sorted event-ID list. AddTo publishes a new +// termState on every write. +// - Dense: dense is the term's denseState. Published once, at +// promotion; later snapshots go into dense.pub. type termState struct { - ids []uint32 - bm *roaring.Bitmap + ids []uint32 + dense *denseState } // ConcurrentBitmaps is the in-memory event index for live ingest: -// one writer, many concurrent readers. Each per-term entry is a -// single atomic.Pointer[termState]. AddTo publishes a new -// termState via a single atomic.Store, so readers atomic.Load and -// operate on an immutable snapshot for as long as they hold the -// pointer. No lock is required across the borrowed pointer's -// lifetime. -// -// The struct-level RWMutex protects only the map's structure (the -// terms map insert when a new key arrives). Once an entry exists, -// all subsequent AddTo and Get operations bypass the lock entirely. +// one writer, many readers. // -// Concurrency contracts: +// rwmu protects only the terms map. Per-term state is reached through +// an atomic pointer. // -// - AddTo: single-writer. The orchestrator drives ingest from one -// goroutine per chunk. The COW Clone inside AddTo (on the dense -// path) mutates the source bitmap's needCopyOnWrite slice as a -// side effect of roaring's clone implementation; two concurrent -// AddTo calls on the same key would race on that. -// -// - Get / LookupKeys: many-reader. No Clone, just atomic.Load. -// Safe to call concurrently with AddTo. +// AddTo is single-writer: the orchestrator ingests from one goroutine +// per chunk. Get is safe to call concurrently with AddTo. Get takes +// the map's read lock for the lookup and no other lock, unless the +// term is dense and was written since its last snapshot; then one +// reader clones the writer's bitmap under the per-term mutex. AddTo +// on a dense term waits for a reader that is cloning that term. type ConcurrentBitmaps struct { rwmu sync.RWMutex terms map[TermKey]*atomic.Pointer[termState] } -// NewConcurrentBitmapsFromBitmaps takes ownership of a single-threaded -// Bitmaps (typically built via warmup or backfill) and wraps it as a -// ConcurrentBitmaps. Each per-term bitmap is marked CopyOnWrite so -// subsequent AddTo calls share containers via lazy COW. +// NewConcurrentBitmapsFromBitmaps takes ownership of a Bitmaps built +// by warmup or backfill. The input must not be used afterwards. // -// The input Bitmaps must not be used after this call returns. +// Terms below promotionThreshold become sparse lists, the same +// representation AddTo gives them. Terms at or above it keep their +// bitmap as the writer-private wbm. No snapshot is published here; +// the first Get on a dense term clones it. func NewConcurrentBitmapsFromBitmaps(b Bitmaps) *ConcurrentBitmaps { cb := &ConcurrentBitmaps{terms: make(map[TermKey]*atomic.Pointer[termState], len(b))} for k, bm := range b { if bm == nil { continue } - bm.SetCopyOnWrite(true) p := &atomic.Pointer[termState]{} - p.Store(&termState{bm: bm}) + if bm.GetCardinality() < promotionThreshold { + p.Store(termStateFromIDs(bm.ToArray())) + } else { + p.Store(&termState{dense: newDenseState(bm)}) + } cb.terms[k] = p } return cb } -// Get returns the bitmap for the given term key, or (nil, nil) when -// the key is not indexed. The returned bitmap is an immutable -// snapshot: writers publish new termStates via atomic.Store, so the -// pointer this method returns will never be mutated by anyone — but -// only if callers respect the "read-only" half of the contract. +// Get returns the bitmap for key, or (nil, nil) when key is not +// indexed. The result is read-only: dense terms share one bitmap +// across all concurrent readers, and the index never mutates it. // -// Forbidden caller-side methods on the returned bitmap (these have -// side effects on the bitmap's internal needCopyOnWrite[] array, -// which the writer's COW Clone also writes to; concurrent calls -// from two goroutines would race): -// -// - Clone, CloneCopyOnWriteContainers +// Forbidden on the returned bitmap — these mutate internal state a +// concurrent reader or the writer may also be touching: +// - Clone, CloneCopyOnWriteContainers (COW is on: Clone writes its +// source's copy-on-write flags, so two readers cloning race) // - RunOptimize, AddRange, RemoveRange, FlipInt // - Add, AddMany, Remove, CheckedAdd, CheckedRemove, AddInt // - SetCopyOnWrite -// - Any *Writable* accessor on the underlying roaringArray -// -// Safe caller-side methods (used by event.Matches today): any -// non-mutating read — Contains, GetCardinality, Iterator, -// ToArray, IsEmpty, Minimum, Maximum — plus the non-mutating -// aggregation entry points roaring.And, roaring.FastAnd (≥2 -// inputs), roaring.FastOr (≥2 inputs), which produce fresh -// result bitmaps without writing through their inputs. Note the -// ≥2-input qualifier on FastAnd/FastOr: with a single input the -// roaring library has historically taken a Clone-the-input -// shortcut, so callers MUST avoid passing a singleton slice to -// those aggregators (the event store's Matches guards its single-input -// cases before calling FastAnd/FastOr). +// - single-input roaring.FastAnd / roaring.FastOr (roaring takes a +// Clone-the-input shortcut when there is only one input) // -// Callers may hold the pointer arbitrarily long. A subsequent Get -// on the same key may return either this same pointer (no AddTo -// happened in between) or a newer snapshot — both are valid; the -// older pointer remains usable until the caller drops it. +// Safe: any non-mutating read (Contains, GetCardinality, Iterator, +// ToArray, IsEmpty, Minimum, Maximum) plus roaring.And / FastAnd / +// FastOr with 2+ inputs. // -// Concurrency: the RLock is held only for the map lookup. Once the -// per-entry pointer is captured, the lock is released; the atomic -// load on the entry happens lock-free. +// A Get that starts after an AddTo returns sees that AddTo's IDs, and +// the pointer stays valid for as long as the caller holds it. func (s *ConcurrentBitmaps) Get(key TermKey) (*roaring.Bitmap, error) { s.rwmu.RLock() p := s.terms[key] @@ -125,34 +109,35 @@ func (s *ConcurrentBitmaps) Get(key TermKey) (*roaring.Bitmap, error) { if p == nil { return nil, nil //nolint:nilnil // not-found is signaled by nil bitmap, no error } - // The pointer is always Stored with a non-nil termState holding a - // non-nil bm or non-empty ids before it is published to the map. st := p.Load() - if st.bm != nil { - return st.bm, nil + if st.dense != nil { + return st.dense.snapshot(), nil } bm := roaring.New() bm.AddMany(st.ids) return bm, nil } -// AddTo records each eventID under key. Idempotent: callers -// (HotStore.applyLedger via the post-commit hook, warmup) feed -// events in chunk-relative event-ID order, so any duplicate is a -// retry of the already-added sorted prefix and is skipped. The same (key, eventID) pair has -// the same effect added once or many times. -// -// Single-writer contract: AddTo must not run concurrently with -// itself. The orchestrator drives ingest from one goroutine per -// chunk. -// -// COW behavior: for an existing dense term, AddTo Clones the -// current bitmap (O(1) shallow copy because the source has -// CopyOnWrite enabled), AddManys the new IDs (which COW-clones only -// the touched containers), then atomic.Stores a new termState -// pointing at the resulting bitmap. The old termState's bitmap is -// untouched — concurrent readers holding it see the previous -// snapshot. +// snapshot returns the term's current immutable bitmap. If a write +// landed since the last snapshot it clones wbm once and publishes +// the clone. +func (d *denseState) snapshot() *roaring.Bitmap { + if bm := d.pub.Load(); bm != nil { + return bm + } + d.mu.Lock() + defer d.mu.Unlock() + bm := d.pub.Load() + if bm == nil { + bm = d.wbm.Clone() + d.pub.Store(bm) + } + return bm +} + +// AddTo records each eventID under key. Callers feed events in +// event-ID order relative to the chunk, so a duplicate is a retry of an +// already-added prefix and is skipped. func (s *ConcurrentBitmaps) AddTo(key TermKey, eventIDs ...uint32) { if len(eventIDs) == 0 { return @@ -173,59 +158,61 @@ func (s *ConcurrentBitmaps) AddTo(key TermKey, eventIDs ...uint32) { } old := p.Load() - if old.bm != nil { - // Dense mode: COW Clone + AddMany + atomic publish. - next := old.bm.Clone() - next.AddMany(eventIDs) - p.Store(&termState{bm: next}) + if d := old.dense; d != nil { + d.mu.Lock() + defer d.mu.Unlock() + d.wbm.AddMany(eventIDs) + d.pub.Store(nil) return } - // Sparse mode: build a new id list (dedup against monotonic - // prefix); promote to dense if we cross the threshold. + // Sparse mode: new list, then promote if it crossed the threshold. ids := make([]uint32, 0, len(old.ids)+len(eventIDs)) ids = append(ids, old.ids...) - for _, id := range eventIDs { - if len(ids) > 0 && ids[len(ids)-1] >= id { + p.Store(termStateFromIDs(appendSorted(ids, eventIDs))) +} + +// appendSorted appends the ids in src that are greater than dst's +// last element. +func appendSorted(dst, src []uint32) []uint32 { + for _, id := range src { + if len(dst) > 0 && dst[len(dst)-1] >= id { continue } - ids = append(ids, id) + dst = append(dst, id) } + return dst +} + +// termStateFromIDs builds the termState for an ascending, unique id +// list, applying promotionThreshold. +func termStateFromIDs(ids []uint32) *termState { if len(ids) >= promotionThreshold { - bm := roaring.New() - bm.AddMany(ids) - // Enable lazy container-level copy-on-write. Subsequent - // AddTo calls clone bm via roaring.Clone (O(1) shallow: - // shares container pointers and marks both bitmaps as - // needing COW), then AddMany routes through - // getWritableContainerAtIndex which deep-copies only the - // touched containers. This turns each ingest call's - // Clone+AddMany into per-touched-container work instead of - // per-bitmap work — ~40× speedup for popular dense terms. - bm.SetCopyOnWrite(true) - p.Store(&termState{bm: bm}) - return + return promote(ids) } - p.Store(&termState{ids: ids}) + return &termState{ids: ids} +} + +// newDenseState wraps a writer-owned bitmap as a dense term. The +// bitmap is marked CopyOnWrite so snapshot's Clone is shallow. No +// snapshot is published; the first Get clones. +func newDenseState(bm *roaring.Bitmap) *denseState { + bm.SetCopyOnWrite(true) + return &denseState{wbm: bm} } -// newTermState builds a fresh termState seeded with the given -// initial eventIDs. Used by AddTo on the new-key path. Promotes to -// dense mode immediately if the initial batch already exceeds the -// threshold. +// promote builds the dense termState for a term from its sorted ids. +func promote(ids []uint32) *termState { + bm := roaring.New() + bm.AddMany(ids) + return &termState{dense: newDenseState(bm)} +} + +// newTermState builds the first termState for a new key. A batch at +// or above the threshold is promoted without building a list. func newTermState(eventIDs []uint32) *termState { if len(eventIDs) >= promotionThreshold { - bm := roaring.New() - bm.AddMany(eventIDs) - bm.SetCopyOnWrite(true) // see AddTo for rationale - return &termState{bm: bm} + return promote(eventIDs) } - ids := make([]uint32, 0, len(eventIDs)) - for _, id := range eventIDs { - if len(ids) > 0 && ids[len(ids)-1] >= id { - continue - } - ids = append(ids, id) - } - return &termState{ids: ids} + return termStateFromIDs(appendSorted(make([]uint32, 0, len(eventIDs)), eventIDs)) } diff --git a/cmd/stellar-rpc/internal/rpcv2/stores/event/concurrent_bitmaps_test.go b/cmd/stellar-rpc/internal/rpcv2/stores/event/concurrent_bitmaps_test.go index fc31d6e8e..2234c5d33 100644 --- a/cmd/stellar-rpc/internal/rpcv2/stores/event/concurrent_bitmaps_test.go +++ b/cmd/stellar-rpc/internal/rpcv2/stores/event/concurrent_bitmaps_test.go @@ -52,7 +52,7 @@ func TestConcurrentBitmaps_ListMode(t *testing.T) { require.NotNil(t, p) st := p.Load() require.NotNil(t, st) - assert.Nil(t, st.bm, "must still be in list mode") + assert.Nil(t, st.dense, "must still be in list mode") assert.Len(t, st.ids, promotionThreshold-1) bm, err := s.Get(key) @@ -61,7 +61,7 @@ func TestConcurrentBitmaps_ListMode(t *testing.T) { assert.Equal(t, uint64(promotionThreshold-1), bm.GetCardinality()) // Get must not have promoted. - assert.Nil(t, p.Load().bm) + assert.Nil(t, p.Load().dense) } func TestConcurrentBitmaps_Promotion(t *testing.T) { @@ -75,9 +75,11 @@ func TestConcurrentBitmaps_Promotion(t *testing.T) { p := s.terms[key] require.NotNil(t, p) st := p.Load() - require.NotNil(t, st.bm) + require.NotNil(t, st.dense) assert.Empty(t, st.ids, "sparse ids cleared after promotion") - assert.Equal(t, uint64(promotionThreshold), st.bm.GetCardinality()) + bm, err := s.Get(key) + require.NoError(t, err) + assert.Equal(t, uint64(promotionThreshold), bm.GetCardinality()) } func TestConcurrentBitmaps_AddAfterPromotion(t *testing.T) { @@ -125,8 +127,10 @@ func TestConcurrentBitmaps_BatchAddToPromotion(t *testing.T) { p := s.terms[key] require.NotNil(t, p) st := p.Load() - require.NotNil(t, st.bm, "single-batch over threshold must promote immediately") - assert.Equal(t, uint64(promotionThreshold+10), st.bm.GetCardinality()) + require.NotNil(t, st.dense, "single-batch over threshold must promote immediately") + bm, err := s.Get(key) + require.NoError(t, err) + assert.Equal(t, uint64(promotionThreshold+10), bm.GetCardinality()) } // TestConcurrentBitmaps_GetReturnsImmutableSnapshot pins the @@ -358,7 +362,7 @@ func TestConcurrentBitmaps_AddToIsIdempotent(t *testing.T) { s.AddTo(key, i) } p := s.terms[key] - require.NotNil(t, p.Load().bm, "must have promoted to bitmap mode") + require.NotNil(t, p.Load().dense, "must have promoted to bitmap mode") // Replay — bitmap.AddMany is set-semantic, so no cardinality change. for i := range uint32(promotionThreshold) { @@ -371,12 +375,29 @@ func TestConcurrentBitmaps_AddToIsIdempotent(t *testing.T) { }) } +// assertDenseCOW checks that a term is dense and that CopyOnWrite is +// enabled on both halves of its state: the writer-private bitmap +// (where it makes the reader-side Clone shallow) and the snapshot +// handed to readers (which inherits the flag through that Clone). +func assertDenseCOW(t *testing.T, s *ConcurrentBitmaps, key TermKey, msg string) { + t.Helper() + p := s.terms[key] + require.NotNil(t, p) + d := p.Load().dense + require.NotNil(t, d, "term must be dense") + assert.True(t, d.wbm.GetCopyOnWrite(), msg+" (writer bitmap)") + bm, err := s.Get(key) + require.NoError(t, err) + require.NotNil(t, bm) + assert.True(t, bm.GetCopyOnWrite(), msg+" (published snapshot)") +} + // TestConcurrentBitmaps_DenseAddToSetsCopyOnWrite pins that the // dense path on AddTo (both the promotion transition in AddTo // itself and newTermState's over-threshold initial batch) sets -// CopyOnWrite on the published bitmap. The perf design relies on -// this: a regression that drops SetCopyOnWrite would silently make -// every subsequent AddTo's Clone deep-copy the whole bitmap +// CopyOnWrite. The perf design relies on this: a regression that +// drops SetCopyOnWrite would silently make every reader-side Clone +// deep-copy the whole bitmap // (+40% hot-ingest wall, observed empirically). func TestConcurrentBitmaps_DenseAddToSetsCopyOnWrite(t *testing.T) { t.Run("via promotion in AddTo", func(t *testing.T) { @@ -385,9 +406,7 @@ func TestConcurrentBitmaps_DenseAddToSetsCopyOnWrite(t *testing.T) { for i := range uint32(promotionThreshold) { s.AddTo(key, i) } - bm := s.terms[key].Load().bm - require.NotNil(t, bm) - assert.True(t, bm.GetCopyOnWrite(), + assertDenseCOW(t, s, key, "dense bitmap after promotion must have CopyOnWrite enabled") }) @@ -399,9 +418,7 @@ func TestConcurrentBitmaps_DenseAddToSetsCopyOnWrite(t *testing.T) { ids[i] = uint32(i) } s.AddTo(key, ids...) - bm := s.terms[key].Load().bm - require.NotNil(t, bm) - assert.True(t, bm.GetCopyOnWrite(), + assertDenseCOW(t, s, key, "dense bitmap from over-threshold initial AddTo must have CopyOnWrite enabled") }) @@ -412,9 +429,7 @@ func TestConcurrentBitmaps_DenseAddToSetsCopyOnWrite(t *testing.T) { s.AddTo(key, i) } s.AddTo(key, 10_000, 10_001, 10_002) - bm := s.terms[key].Load().bm - require.NotNil(t, bm) - assert.True(t, bm.GetCopyOnWrite(), + assertDenseCOW(t, s, key, "dense bitmap after additional AddTos must keep CopyOnWrite (inherited via Clone)") }) } @@ -423,46 +438,49 @@ func TestConcurrentBitmaps_DenseAddToSetsCopyOnWrite(t *testing.T) { // the warmup-side conversion constructor: // - input bitmaps survive in the result with their cardinality // intact; -// - each non-nil bitmap has CopyOnWrite enabled after conversion; -// - nil bitmaps in the input map are skipped (not panicked); -// - subsequent AddTo on the result Clones via the COW fast path -// (no full deep copy on the popular-term ingest path). +// - terms below promotionThreshold become sparse lists, terms at or +// above it become dense and keep CopyOnWrite; +// - nil bitmaps in the input map are skipped; +// - a subsequent AddTo on a converted dense term is visible to the +// next Get. func TestNewConcurrentBitmapsFromBitmaps_DirectlyPinsContract(t *testing.T) { src := NewBitmaps() - keyA := ComputeTermKey([]byte("a"), FieldTopic0) - keyB := ComputeTermKey([]byte("b"), FieldTopic1) + keySparse := ComputeTermKey([]byte("a"), FieldTopic0) + keyDense := ComputeTermKey([]byte("b"), FieldTopic1) keyNil := ComputeTermKey([]byte("nil"), FieldTopic2) - src.AddTo(keyA, 0, 1, 2, 3, 4) - src.AddTo(keyB, 100, 101) - src[keyNil] = nil // simulate a nil entry that the constructor must skip + src.AddTo(keySparse, 0, 1, 2, 3, 4) + dense := make([]uint32, promotionThreshold) + for i := range dense { + dense[i] = uint32(i) * 1_000 + } + src.AddTo(keyDense, dense...) + src[keyNil] = nil cb := NewConcurrentBitmapsFromBitmaps(src) - bmA, err := cb.Get(keyA) + bmSparse, err := cb.Get(keySparse) require.NoError(t, err) - require.NotNil(t, bmA) - assert.Equal(t, uint64(5), bmA.GetCardinality()) - assert.True(t, bmA.GetCopyOnWrite(), - "FromBitmaps must enable CopyOnWrite so the live-ingest AddTo path Clones via the fast shallow path") + require.NotNil(t, bmSparse) + assert.Equal(t, uint64(5), bmSparse.GetCardinality()) + assert.Nil(t, cb.terms[keySparse].Load().dense, "a sub-threshold warmup term must be sparse") - bmB, err := cb.Get(keyB) + bmDense, err := cb.Get(keyDense) require.NoError(t, err) - require.NotNil(t, bmB) - assert.Equal(t, uint64(2), bmB.GetCardinality()) - assert.True(t, bmB.GetCopyOnWrite()) + require.NotNil(t, bmDense) + assert.Equal(t, uint64(len(dense)), bmDense.GetCardinality()) + assert.True(t, bmDense.GetCopyOnWrite()) + assert.NotNil(t, cb.terms[keyDense].Load().dense, "a warmup term at the threshold must be dense") bmNil, err := cb.Get(keyNil) require.NoError(t, err) assert.Nil(t, bmNil, "nil source entries must be skipped, not panicked") - // Subsequent AddTo on the converted index produces a new - // termState whose bitmap still has CopyOnWrite (inherited via - // Clone). This pins that the AddTo dense path doesn't lose the - // COW flag. - cb.AddTo(keyA, 5) - post := cb.terms[keyA].Load().bm + cb.AddTo(keyDense, 999_999) + post, err := cb.Get(keyDense) + require.NoError(t, err) require.NotNil(t, post) assert.True(t, post.GetCopyOnWrite()) - assert.Equal(t, uint64(6), post.GetCardinality()) + assert.Equal(t, uint64(len(dense)+1), post.GetCardinality()) + assert.Equal(t, uint64(len(dense)), bmDense.GetCardinality(), "earlier snapshot stays immutable") } diff --git a/cmd/stellar-rpc/internal/rpcv2/stores/event/concurrent_bitmaps_writer_test.go b/cmd/stellar-rpc/internal/rpcv2/stores/event/concurrent_bitmaps_writer_test.go new file mode 100644 index 000000000..cfd493caa --- /dev/null +++ b/cmd/stellar-rpc/internal/rpcv2/stores/event/concurrent_bitmaps_writer_test.go @@ -0,0 +1,498 @@ +package event + +import ( + "runtime" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestConcurrentBitmaps_SnapshotSurvivesMultiContainerWrites: a +// snapshot handed to a reader is never mutated afterwards. The ids +// span ~16 containers so the writer both deep-copies shared +// containers and inserts new ones. +func TestConcurrentBitmaps_SnapshotSurvivesMultiContainerWrites(t *testing.T) { + s := newTestConcurrentBitmaps() + key := ComputeTermKey([]byte("multi-container"), FieldTopic0) + + // Promote and seed one container. + seed := make([]uint32, promotionThreshold) + for i := range seed { + seed[i] = uint32(i) + } + s.AddTo(key, seed...) + + held, err := s.Get(key) + require.NoError(t, err) + require.NotNil(t, held) + heldCard := held.GetCardinality() + require.Equal(t, uint64(promotionThreshold), heldCard) + heldCopy := held.ToArray() + + // 1_000 more ids spread over 16 fresh containers, written one + // small batch per "ledger" so the writer touches an already-shared + // container repeatedly. + added := make([]uint32, 0, 100*10) + for batch := range 100 { + ids := make([]uint32, 10) + for j := range ids { + ids[j] = uint32(batch%16)*65_536 + 1_000 + uint32(batch)*10 + uint32(j) + } + s.AddTo(key, ids...) + added = append(added, ids...) + } + + assert.Equal(t, heldCard, held.GetCardinality(), + "a snapshot returned by Get must not be mutated by later AddTo calls") + assert.Equal(t, heldCopy, held.ToArray(), + "a snapshot returned by Get must keep exactly its original ids") + for _, id := range added { + assert.False(t, held.Contains(id), + "the held snapshot must not observe ids added after it was taken") + } + + fresh, err := s.Get(key) + require.NoError(t, err) + require.NotNil(t, fresh) + assert.Equal(t, heldCard+uint64(len(added)), fresh.GetCardinality(), + "a Get after AddTo must observe every write") + for _, id := range added { + require.True(t, fresh.Contains(id), "fresh snapshot missing id %d", id) + } +} + +// TestConcurrentBitmaps_GetAfterAddToIsFresh pins the freshness +// contract call by call: every single AddTo must be visible to the +// very next Get. The published snapshot is allowed to lag, but only +// until someone reads. +func TestConcurrentBitmaps_GetAfterAddToIsFresh(t *testing.T) { + s := newTestConcurrentBitmaps() + key := ComputeTermKey([]byte("freshness"), FieldTopic0) + + for i := range uint32(promotionThreshold + 500) { + s.AddTo(key, i) + bm, err := s.Get(key) + require.NoError(t, err) + require.NotNil(t, bm) + require.Equal(t, uint64(i+1), bm.GetCardinality(), + "Get after AddTo(%d) must see every id added so far", i) + require.True(t, bm.Contains(i)) + } +} + +// TestConcurrentBitmaps_WriterReaderRace runs one writer against +// eight readers that both snapshot and read through the snapshot +// (Contains / GetCardinality / Iterator). Under -race this pins that +// the writer's in-place AddMany and the readers' Clone never overlap +// on the writer-private bitmap, and that nothing mutates a snapshot +// while another goroutine reads it. The writer starts only once every +// reader is in its loop, and each reader keeps going for minReads +// iterations past the writer's end, so the test cannot pass without +// reader work overlapping writes. +func TestConcurrentBitmaps_WriterReaderRace(t *testing.T) { + s := newTestConcurrentBitmaps() + + const numTerms = 32 + const numLedgers = 400 + const perLedger = 8 + const numReaders = 8 + const minReads = 64 + + keys := make([]TermKey, numTerms) + for i := range keys { + keys[i] = ComputeTermKey([]byte{0xAA, byte(i)}, FieldTopic1) + } + + var done atomic.Bool + var reads atomic.Uint64 + var ready, wg sync.WaitGroup + ready.Add(numReaders) + + wg.Go(func() { + defer done.Store(true) + ready.Wait() + var next uint32 + for range numLedgers { + for _, key := range keys { + ids := make([]uint32, perLedger) + for j := range ids { + ids[j] = next + next++ + } + s.AddTo(key, ids...) + } + } + }) + + for r := range numReaders { + wg.Go(func() { + ready.Done() + for i, n := 0, 0; !done.Load() || n < minReads; i++ { + key := keys[(i+r)%numTerms] + bm, err := s.Get(key) + if err != nil || bm == nil { + continue + } + n++ + reads.Add(1) + _ = bm.GetCardinality() + _ = bm.Contains(uint32(i)) + it := bm.Iterator() + for n := 0; n < 64 && it.HasNext(); n++ { + _ = it.Next() + } + } + }) + } + + wg.Wait() + assert.GreaterOrEqual(t, reads.Load(), uint64(numReaders*minReads), + "every reader must have done real reads") + + want := uint64(numLedgers * perLedger) + for _, key := range keys { + bm, err := s.Get(key) + require.NoError(t, err) + require.NotNil(t, bm) + assert.Equal(t, want, bm.GetCardinality()) + } +} + +// TestConcurrentBitmaps_PromotionMidStreamUnderReaders drives the +// sparse→dense transition while readers are live, then keeps writing +// past it. Readers must never see a nil bitmap, never see a +// cardinality that goes backwards, and must see the full set at the +// end. +func TestConcurrentBitmaps_PromotionMidStreamUnderReaders(t *testing.T) { + s := newTestConcurrentBitmaps() + key := ComputeTermKey([]byte("promote-midstream"), FieldTopic2) + + const total = promotionThreshold * 8 + const numReaders = 4 + const minReads = 64 + + var writes atomic.Uint32 + var done atomic.Bool + var reads atomic.Uint64 + var ready, wg sync.WaitGroup + ready.Add(numReaders) + + wg.Go(func() { + defer done.Store(true) + ready.Wait() + for i := range uint32(total) { + s.AddTo(key, i) + writes.Store(i + 1) + } + }) + + for range numReaders { + wg.Go(func() { + ready.Done() + var last uint64 + for n := 0; !done.Load() || n < minReads; { + bm, err := s.Get(key) + if err != nil || bm == nil { + continue + } + n++ + reads.Add(1) + card := bm.GetCardinality() + assert.GreaterOrEqual(t, card, last, + "a term's observed cardinality must never go backwards") + assert.LessOrEqual(t, card, uint64(writes.Load())+1, + "a reader must not observe more ids than were written") + last = card + } + }) + } + + wg.Wait() + assert.GreaterOrEqual(t, reads.Load(), uint64(numReaders*minReads), + "every reader must have done real reads") + + bm, err := s.Get(key) + require.NoError(t, err) + require.NotNil(t, bm) + assert.Equal(t, uint64(total), bm.GetCardinality()) +} + +// TestConcurrentBitmaps_WarmupThenAddToThenGet covers the warmup +// entry path: NewConcurrentBitmapsFromBitmaps takes ownership of the +// built bitmaps without publishing a snapshot, so the first Get has +// to materialize one. A write before that first read must still be +// visible. +func TestConcurrentBitmaps_WarmupThenAddToThenGet(t *testing.T) { + src := NewBitmaps() + keyRead := ComputeTermKey([]byte("warm-read-first"), FieldTopic0) + keyWrite := ComputeTermKey([]byte("warm-write-first"), FieldTopic0) + + seed := make([]uint32, 200) + for i := range seed { + seed[i] = uint32(i) * 1_000 // spans several containers + } + src.AddTo(keyRead, seed...) + src.AddTo(keyWrite, seed...) + + cb := NewConcurrentBitmapsFromBitmaps(src) + + // Read first, then write, then read again. + first, err := cb.Get(keyRead) + require.NoError(t, err) + require.NotNil(t, first) + require.Equal(t, uint64(len(seed)), first.GetCardinality()) + cb.AddTo(keyRead, 999_999) + assert.Equal(t, uint64(len(seed)), first.GetCardinality(), + "the warmup snapshot handed out earlier must stay immutable") + second, err := cb.Get(keyRead) + require.NoError(t, err) + assert.Equal(t, uint64(len(seed)+1), second.GetCardinality()) + assert.True(t, second.Contains(999_999)) + + // Write before the first read ever happens. + cb.AddTo(keyWrite, 888_888) + only, err := cb.Get(keyWrite) + require.NoError(t, err) + require.NotNil(t, only) + assert.Equal(t, uint64(len(seed)+1), only.GetCardinality()) + assert.True(t, only.Contains(888_888)) + assert.True(t, only.GetCopyOnWrite(), + "warmup snapshots must keep CopyOnWrite so republishing stays shallow") +} + +// TestConcurrentBitmaps_UnreadTermNeverClones pins the allocation +// property the fix exists for: a dense term that nobody reads +// publishes no snapshot at all, so neither the entry's termState nor +// the term's published bitmap moves across thousands of AddTo calls. +// A regression that re-published per AddTo would move one of them. +func TestConcurrentBitmaps_UnreadTermNeverClones(t *testing.T) { + s := newTestConcurrentBitmaps() + key := ComputeTermKey([]byte("unread"), FieldTopic0) + + seed := make([]uint32, promotionThreshold) + for i := range seed { + seed[i] = uint32(i) + } + s.AddTo(key, seed...) + + entry := s.terms[key].Load() + d := entry.dense + require.NotNil(t, d) + require.Nil(t, d.pub.Load(), "an unread dense term must publish nothing at promotion") + + for i := range uint32(5_000) { + s.AddTo(key, 100_000+i) + } + assert.Same(t, entry, s.terms[key].Load(), + "AddTo on a dense term must not publish a new termState") + assert.Nil(t, d.pub.Load(), + "AddTo on a dense term must not publish a snapshot") + + bm, err := s.Get(key) + require.NoError(t, err) + assert.Equal(t, uint64(promotionThreshold+5_000), bm.GetCardinality()) + assert.Same(t, entry, s.terms[key].Load(), + "a dense term's termState is published once and never replaced") + assert.Same(t, bm, d.pub.Load(), + "the first Get after writes must publish the snapshot it returns") + + // A second Get with no intervening write returns the same pointer. + again, err := s.Get(key) + require.NoError(t, err) + assert.Same(t, bm, again, + "Get must return the same pointer when nothing changed") +} + +// TestConcurrentBitmaps_DenseStateAfterAddToAndGet checks the state +// of a dense term at each step: no snapshot after promotion and after +// a write, the returned snapshot in pub after a read. Earlier +// snapshots stay frozen. +func TestConcurrentBitmaps_DenseStateAfterAddToAndGet(t *testing.T) { + s := newTestConcurrentBitmaps() + key := ComputeTermKey([]byte("publish-order"), FieldTopic0) + + seed := make([]uint32, promotionThreshold) + for i := range seed { + seed[i] = uint32(i) + } + s.AddTo(key, seed...) + + d := s.terms[key].Load().dense + require.NotNil(t, d) + assert.Nil(t, d.pub.Load(), "a freshly promoted term has no snapshot until first read") + + first, err := s.Get(key) + require.NoError(t, err) + assert.Same(t, first, d.pub.Load(), + "the publishing reader stores the snapshot it returns") + + s.AddTo(key, 12_345) + assert.Nil(t, d.pub.Load(), "AddTo clears the stale snapshot before returning") + assert.False(t, first.Contains(12_345), "an earlier snapshot stays frozen") + + second, err := s.Get(key) + require.NoError(t, err) + assert.True(t, second.Contains(12_345), "the next Get observes the write") + assert.Same(t, second, d.pub.Load()) + assert.False(t, first.Contains(12_345), "the earlier snapshot is still frozen") +} + +// freshnessWriterCounters is the shared state between the freshness +// stress writer and its readers. +type freshnessWriterCounters struct { + committed *atomic.Uint32 // highest ID whose AddTo has returned + observed *atomic.Uint32 // highest committed ID a reader has seen in a snapshot + batches *atomic.Uint32 // batches the writer has finished +} + +// freshnessWriter adds numBatches three-ID batches under key, handing +// off to the readers after each one: it does not start the next batch +// until some reader has returned a snapshot holding this batch's last +// ID. That snapshot must be a clone made after the AddTo, so every +// batch forces at least one republish, and the readers race each +// other to make it. Returns early once the test has failed so a +// reader that gave up cannot hang the writer. +func freshnessWriter( + t *testing.T, s *ConcurrentBitmaps, key TermKey, c freshnessWriterCounters, + numBatches, firstID, idStride uint32, +) { + next := firstID + for i := range numBatches { + batch := []uint32{next, next + 977, next + 65_536} + s.AddTo(key, batch...) + last := batch[len(batch)-1] + c.committed.Store(last) + c.batches.Store(i + 1) + next += idStride + for c.observed.Load() < last { + if t.Failed() { + return + } + runtime.Gosched() + } + } +} + +// storeMax raises v to want unless v is already at least want. +func storeMax(v *atomic.Uint32, want uint32) { + for { + seen := v.Load() + if seen >= want || v.CompareAndSwap(seen, want) { + return + } + } +} + +// TestConcurrentBitmaps_FreshnessUnderConcurrentPublishers: the +// writer publishes the highest ID it has finished adding; each reader +// loads that ID, calls Get, and asserts the snapshot contains it. The +// batch count is fixed so the ids stay below MaxUint32 and never +// repeat an id an older snapshot already holds. +func TestConcurrentBitmaps_FreshnessUnderConcurrentPublishers(t *testing.T) { + s := newTestConcurrentBitmaps() + key := ComputeTermKey([]byte("freshness-stress"), FieldTopic0) + + // Seed ~200 containers so a republish Clone is long enough for a + // concurrent reader to interleave with it. + seed := make([]uint32, 0, 200*8) + for c := range uint32(200) { + for j := range uint32(8) { + seed = append(seed, c*65_536+j) + } + } + s.AddTo(key, seed...) + + numReaders := max(16, 2*runtime.GOMAXPROCS(0)) + // firstID + numBatches*idStride + 65_536 stays below MaxUint32. + const ( + numBatches = 1_000 + firstID = uint32(20_000_000) + idStride = uint32(131_072) + ) + + var committed, observed, batches atomic.Uint32 + var done atomic.Bool + var reads atomic.Uint64 + var wg sync.WaitGroup + + wg.Go(func() { + defer done.Store(true) + freshnessWriter(t, s, key, freshnessWriterCounters{ + committed: &committed, observed: &observed, batches: &batches, + }, numBatches, firstID, idStride) + }) + + for range numReaders { + wg.Go(func() { + for !done.Load() { + want := committed.Load() + if want == 0 { + runtime.Gosched() + continue + } + bm, err := s.Get(key) + if err != nil || bm == nil { + continue + } + reads.Add(1) + // Yield: every read after a write takes the term + // mutex, and unyielding readers starve the writer. + runtime.Gosched() + // Get started after the AddTo that committed `want` + // returned, so the snapshot must contain it. + if !bm.Contains(want) { + t.Errorf("Get returned a snapshot missing id %d, "+ + "committed before this Get started (cardinality %d)", + want, bm.GetCardinality()) + return + } + storeMax(&observed, want) + } + }) + } + + wg.Wait() + t.Logf("freshness stress: %d reads, %d batches", + reads.Load(), batches.Load()) + assert.Equal(t, uint32(numBatches), batches.Load(), "the writer must finish every batch") + assert.GreaterOrEqual(t, observed.Load(), committed.Load(), + "a reader must observe the final committed batch") + require.Positive(t, reads.Load(), "the stress loop must have done real reads") +} + +// TestConcurrentBitmaps_WarmupSubThresholdTermStaysSparse pins the +// warmup representation of a small term: it is a sparse list, Get +// returns its ids, later AddTo calls stay in sparse mode, and the term +// promotes at the threshold like a term built by AddTo. +func TestConcurrentBitmaps_WarmupSubThresholdTermStaysSparse(t *testing.T) { + src := NewBitmaps() + key := ComputeTermKey([]byte("warm-small"), FieldTopic0) + seed := make([]uint32, promotionThreshold-2) + for i := range seed { + seed[i] = uint32(i) * 7 + } + src.AddTo(key, seed...) + + cb := NewConcurrentBitmapsFromBitmaps(src) + st := cb.terms[key].Load() + require.Nil(t, st.dense) + assert.Equal(t, seed, st.ids) + + bm, err := cb.Get(key) + require.NoError(t, err) + assert.Equal(t, uint64(len(seed)), bm.GetCardinality()) + + next := seed[len(seed)-1] + 1 + cb.AddTo(key, next) + require.Nil(t, cb.terms[key].Load().dense, "one below the threshold stays sparse") + + cb.AddTo(key, next+1) + require.NotNil(t, cb.terms[key].Load().dense, "reaching the threshold promotes") + bm, err = cb.Get(key) + require.NoError(t, err) + assert.Equal(t, uint64(len(seed)+2), bm.GetCardinality()) + assert.True(t, bm.Contains(next+1)) +} diff --git a/cmd/stellar-rpc/internal/rpcv2/stores/event/hot_store.go b/cmd/stellar-rpc/internal/rpcv2/stores/event/hot_store.go index f30fa809d..ec2e3730c 100644 --- a/cmd/stellar-rpc/internal/rpcv2/stores/event/hot_store.go +++ b/cmd/stellar-rpc/internal/rpcv2/stores/event/hot_store.go @@ -459,9 +459,9 @@ func (h *HotStore) index() *ConcurrentBitmaps { return h.mirror } // later one. Reversing it would let a reader see an offsets count including IDs // the mirror hasn't published — FetchEvents would then miss them, silently. func (h *HotStore) applyLedger(startID uint32, termKeys [][]TermKey) { - // Batch by key so each AddTo clones at most once per (key, ledger), not per - // (key, event) — turns N COW clones into 1 for popular terms. Cap 64 ≈ a few - // × unique-terms per ledger; the map grows past that. + // Batch by key so each key takes one AddTo per ledger instead of one per + // event, saving per-term lock round-trips. Cap 64 ≈ a few × unique-terms + // per ledger; the map grows past that. perKeyIDs := make(map[TermKey][]uint32, 64) for i, keys := range termKeys { eventID := startID + uint32(i) @@ -567,10 +567,9 @@ func verifyChunkConsistency(chunkStore *rocksdb.Store, total uint32, indexUpperB // per-term batching (rocksdb's byte-sorted iteration delivers all // rows for term K consecutively, so a small buffer flushes when the // term changes), then convert to ConcurrentBitmaps at the end. This -// avoids paying the per-row Clone cost the concurrent ConcurrentBitmaps.AddTo -// would do for popular terms — without batching, warmup of a -// 10M-event chunk does ~50M Clones (one per index row) and saturates -// GC for many minutes. +// keeps warmup off the concurrent index's per-term locking and +// promotion path — one AddTo per term instead of one per index row, +// ~50M for a 10M-event chunk. // // Also returns the exclusive upper bound of indexed event IDs (max + 1, // or 0 if the index is empty; uint64 so max+1 can't wrap — see diff --git a/cmd/stellar-rpc/internal/rpcv2/stores/event/reader.go b/cmd/stellar-rpc/internal/rpcv2/stores/event/reader.go index 9c72fb865..870c7ca88 100644 --- a/cmd/stellar-rpc/internal/rpcv2/stores/event/reader.go +++ b/cmd/stellar-rpc/internal/rpcv2/stores/event/reader.go @@ -98,17 +98,15 @@ type Reader interface { // ColdReader coalesces the underlying packfile reads into a // single ReadItems pass, fanning out across the worker count // configured via ColdReaderOptions.Concurrency. HotStore returns - // borrowed mirror references with no per-key Clone. + // snapshots of the live mirror shared by all readers of a term; + // a dense term written since its last lookup is cloned once, by + // the first reader to look it up, and that clone is then shared. // - // Bitmap ownership: callers MUST treat returned bitmaps as - // read-only. The hot path returns immutable snapshots of the - // live mirror — ConcurrentBitmaps stores bitmap pointers via - // atomic.Pointer COW, so a returned pointer will never be - // mutated by anyone. The cold path returns freshly-unmarshaled - // bitmaps logically owned by the caller. Either way callers - // must not mutate; event.Matches is the only consumer today - // and never mutates, and downstream roaring.FastAnd/FastOr never - // mutate inputs. + // Callers MUST treat returned bitmaps as read-only. Dense hot + // snapshots are shared with other readers; sparse hot terms + // return a fresh caller-owned bitmap per lookup; cold-path + // bitmaps are freshly unmarshaled and owned by the caller. See + // ConcurrentBitmaps.Get. // // ctx cancels in-flight I/O on the cold path (MPHF load, // index.pack ReadAt); hot side checks ctx as a fast guard before