From d0f502f5b0706b9625956f4fb00a2e9683d79662 Mon Sep 17 00:00:00 2001 From: tamirms Date: Tue, 6 Oct 2026 09:49:28 +0100 Subject: [PATCH] storage: keep the window index being rebuilt and build scratch with the hot tier The tx-hash index of the window still being filled is rewritten at every chunk boundary until the window terminates, so it is the one cold artifact that is not write-once: on a throughput-capped volume the rewrite competes with every cold read for as long as it runs. It now lives with the hot tier: a coverage that is not terminal is written under the hot root, in a txhash-index-{window} directory beside the chunk DBs, and the terminal coverage is a cold artifact under the txhash index root, decided from the coverage alone in Layout.TxHashIndexDir. The build, the sweep and the reader all resolve the path from the coverage, so the terminal build lands in the cold root and the same commit batch demotes the last hot-tier coverage for the sweep. The layout now owns the window arithmetic, which the catalog takes from it, so the two cannot disagree. The events index build's spilled runs move from the index root to a scratch directory under the hot root, so the cold roots hold only artifacts and moving the hot root moves every build's transient writes with it. The sample config names what belongs on the local disk (hot, txhash_raw, captive core) and that the catalog stays on the durable volume. Co-Authored-By: Claude Fable 5.1 --- .../internal/rpcv2/backfill/txindex.go | 2 +- .../internal/rpcv2/backfill/txindex_test.go | 18 +++++- .../internal/rpcv2/bench/bench_test.go | 3 +- .../internal/rpcv2/bench/dataset.go | 2 +- .../internal/rpcv2/catalog/catalog.go | 25 ++++----- .../rpcv2/catalog/catalog_protocol.go | 2 +- .../rpcv2/catalog/catalog_protocol_test.go | 6 +- .../internal/rpcv2/catalog/catalog_sweep.go | 2 +- .../rpcv2/catalog/catalog_sweep_test.go | 2 + .../internal/rpcv2/catalog/census.go | 4 +- .../rpcv2/catalog/crashsafety_test.go | 7 +++ .../internal/rpcv2/catalog/helpers_test.go | 17 +++--- .../internal/rpcv2/catalog/kv_test.go | 11 +--- .../internal/rpcv2/config/config_test.go | 4 +- .../rpcv2/config/rpc-v2-sample-config.toml | 11 ++-- cmd/stellar-rpc/internal/rpcv2/daemon.go | 2 +- cmd/stellar-rpc/internal/rpcv2/daemon_test.go | 5 +- cmd/stellar-rpc/internal/rpcv2/e2e_test.go | 2 +- .../internal/rpcv2/geometry/keys_test.go | 20 ++++++- .../internal/rpcv2/geometry/paths.go | 55 ++++++++++++++----- .../internal/rpcv2/ingest/driver.go | 2 +- .../internal/rpcv2/ingest/events.go | 4 +- .../internal/rpcv2/ingest/ingest_test.go | 7 ++- .../internal/rpcv2/rpcv2test/rpcv2test.go | 2 +- .../internal/rpcv2/stores/event/cold_dirs.go | 13 +++-- .../rpcv2/stores/event/cold_format.go | 2 +- .../internal/rpcv2/stores/event/cold_index.go | 16 +++--- .../rpcv2/stores/event/cold_index_test.go | 16 +++--- .../full-history-streaming-workflow.md | 16 +++--- design-docs/getevents-full-history-design.md | 11 ++-- 30 files changed, 173 insertions(+), 116 deletions(-) diff --git a/cmd/stellar-rpc/internal/rpcv2/backfill/txindex.go b/cmd/stellar-rpc/internal/rpcv2/backfill/txindex.go index a60b42260..accb396a8 100644 --- a/cmd/stellar-rpc/internal/rpcv2/backfill/txindex.go +++ b/cmd/stellar-rpc/internal/rpcv2/backfill/txindex.go @@ -90,7 +90,7 @@ func buildTxhashIndex(ctx context.Context, w geometry.TxHashIndexID, lo, hi chun // one-write:create — write from scratch (BuildColdIndex truncates any crashed // partial). MkdirAll is idempotent, so the index dir is created on demand. - indexDir := layout.TxHashIndexDir(w) + indexDir := layout.TxHashIndexDir(cov) if mkErr := os.MkdirAll(indexDir, 0o755); mkErr != nil { return fmt.Errorf("buildTxhashIndex mkdir %s: %w", indexDir, mkErr) } diff --git a/cmd/stellar-rpc/internal/rpcv2/backfill/txindex_test.go b/cmd/stellar-rpc/internal/rpcv2/backfill/txindex_test.go index edbe1c435..fd30f022b 100644 --- a/cmd/stellar-rpc/internal/rpcv2/backfill/txindex_test.go +++ b/cmd/stellar-rpc/internal/rpcv2/backfill/txindex_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "sort" + "strings" "testing" "github.com/stretchr/testify/require" @@ -210,6 +211,8 @@ func TestBuildThenSweep_RollingPredecessorDemotedNotSwept(t *testing.T) { require.True(t, ok) require.Equal(t, chunk.ID(0), frozen.Lo) require.Equal(t, hi, frozen.Hi) + require.True(t, strings.HasPrefix(cat.Layout().TxHashIndexFilePath(frozen), cat.Layout().HotRoot()), + "a window still filling keeps its index with the hot tier") // The predecessors linger as "pruning" debris: the old .idx is // query-visible, so its deletion waits for a lifecycle run's grace- @@ -279,6 +282,8 @@ func TestBuildThenSweep_TerminalDemotesAndSweepsAllInputs(t *testing.T) { require.True(t, ok) require.True(t, cat.TxHashIndexLayout().IsTerminalCoverage(frozen)) require.Equal(t, chunk.ID(3), frozen.Hi) + require.True(t, strings.HasPrefix(cat.Layout().TxHashIndexFilePath(frozen), cat.Layout().TxHashIndexRoot()), + "a terminal index is a cold artifact") // Every in-window txhash key was demoted AND swept: key absent => .bin gone. for c := chunk.ID(0); c <= 3; c++ { @@ -462,7 +467,8 @@ func TestBuildCrashMatrix_AfterCommitBeforeSweep(t *testing.T) { } // A predecessor [0,2] so the commit has a coverage to demote too. require.NoError(t, buildThenSweep(context.Background(), IndexBuild{Index: 0, Lo: 0, Hi: 2}, cfg)) - predPath := cat.Layout().TxHashIndexFilePath(geometry.TxHashIndexCoverage{Index: 0, Lo: 0, Hi: 2}) + pred := geometry.TxHashIndexCoverage{Index: 0, Lo: 0, Hi: 2, Key: geometry.TxHashIndexKey(0, 0, 2)} + predPath := cat.Layout().TxHashIndexFilePath(pred) // Reconstruct the "crash after commit, before the eager sweep" of the terminal // build [0,3]: buildTxhashIndex commits WITHOUT sweeping (the sweep is @@ -481,7 +487,7 @@ func TestBuildCrashMatrix_AfterCommitBeforeSweep(t *testing.T) { for _, k := range keys { states[k.Key] = k.State } - require.Equal(t, geometry.StatePruning, states[geometry.TxHashIndexKey(0, 0, 2)], "predecessor demoted, not yet swept") + require.Equal(t, geometry.StatePruning, states[pred.Key], "predecessor demoted, not yet swept") for c := chunk.ID(0); c <= 3; c++ { s, serr := cat.State(c, geometry.KindTxHash) require.NoError(t, serr) @@ -513,7 +519,13 @@ func TestBuildCrashMatrix_AfterCommitBeforeSweep(t *testing.T) { states[k.Key] = k.State } require.Equal(t, geometry.StateFrozen, states[geometry.TxHashIndexKey(0, 0, 3)]) - require.Equal(t, geometry.StatePruning, states[geometry.TxHashIndexKey(0, 0, 2)]) + require.Equal(t, geometry.StatePruning, states[pred.Key]) + + // The lifecycle's destroy takes the predecessor's file and its hot-tier + // directory; the terminal index stays in the cold root. + require.NoError(t, cat.DestroyTxHashIndexKey(pred)) + require.NoDirExists(t, cat.Layout().TxHashIndexDir(pred)) + require.FileExists(t, cat.Layout().TxHashIndexFilePath(frozen)) } // Row "mid-sweep": a "pruning" key whose durable unlink completed but whose key diff --git a/cmd/stellar-rpc/internal/rpcv2/bench/bench_test.go b/cmd/stellar-rpc/internal/rpcv2/bench/bench_test.go index 9fa7bf1a4..02536b4e3 100644 --- a/cmd/stellar-rpc/internal/rpcv2/bench/bench_test.go +++ b/cmd/stellar-rpc/internal/rpcv2/bench/bench_test.go @@ -91,8 +91,7 @@ func readCSV(t *testing.T, path string) map[string]map[string]int64 { // index .idx (both chunks inside one window, as every test range here is). func txhashIndexPath(t *testing.T, layout geometry.Layout, lo, hi chunk.ID) string { t.Helper() - txLayout, err := geometry.NewTxHashIndexLayout(geometry.ChunksPerTxhashIndex) - require.NoError(t, err) + txLayout := layout.TxHashIndex() w := txLayout.TxHashIndexID(lo) require.Equal(t, w, txLayout.TxHashIndexID(hi), "test range must stay inside one index window") return layout.TxHashIndexFilePath(geometry.TxHashIndexCoverage{Index: w, Lo: lo, Hi: hi}) diff --git a/cmd/stellar-rpc/internal/rpcv2/bench/dataset.go b/cmd/stellar-rpc/internal/rpcv2/bench/dataset.go index 9b2621cfe..e6b77268c 100644 --- a/cmd/stellar-rpc/internal/rpcv2/bench/dataset.go +++ b/cmd/stellar-rpc/internal/rpcv2/bench/dataset.go @@ -30,7 +30,7 @@ func openDatasetCatalog(layout geometry.Layout, logger *supportlog.Entry) (*cata if err != nil { return nil, err } - cat, err := catalog.Open(layout.CatalogPath(), layout, txLayout, logger) + cat, err := catalog.Open(layout.CatalogPath(), layout.WithTxHashIndex(txLayout), logger) if err != nil { return nil, fmt.Errorf("open dataset catalog %s: %w", layout.CatalogPath(), err) } diff --git a/cmd/stellar-rpc/internal/rpcv2/catalog/catalog.go b/cmd/stellar-rpc/internal/rpcv2/catalog/catalog.go index a1142a5e3..1ff58cb17 100644 --- a/cmd/stellar-rpc/internal/rpcv2/catalog/catalog.go +++ b/cmd/stellar-rpc/internal/rpcv2/catalog/catalog.go @@ -27,25 +27,22 @@ import ( // concurrency guard: the design's Concurrency model guarantees one writer per // key (see the header note in catalog_protocol.go). type Catalog struct { - store *rocksdb.Store - logger *supportlog.Entry - layout geometry.Layout - txhashIndex geometry.TxHashIndexLayout - secret [32]byte // cold-index secret, minted once at Open then read-only + store *rocksdb.Store + logger *supportlog.Entry + layout geometry.Layout + secret [32]byte // cold-index secret, minted once at Open then read-only } // Open opens the catalog's backing KV store at path (created if absent) and -// binds the catalog to it, the on-disk layout, and the tx-hash-index -// arithmetic. path and logger are required (rocksdb.New validates both). The -// catalog owns the store: Close releases it. -func Open( - path string, layout geometry.Layout, txhashIndex geometry.TxHashIndexLayout, logger *supportlog.Entry, -) (*Catalog, error) { +// binds the catalog to it and the on-disk layout, whose tx-hash index +// arithmetic it uses. path and logger are required (rocksdb.New validates +// both). The catalog owns the store: Close releases it. +func Open(path string, layout geometry.Layout, logger *supportlog.Entry) (*Catalog, error) { store, err := rocksdb.New(rocksdb.Config{Path: path, Logger: logger}) if err != nil { return nil, err } - c := &Catalog{store: store, logger: logger, layout: layout, txhashIndex: txhashIndex} + c := &Catalog{store: store, logger: logger, layout: layout} // Census before the secret mint below: a catalog holding entries outside // this binary's vocabulary (a newer binary's formats, or corruption) is // refused here, so Open writes no catalog entry of its own into a tree it @@ -116,7 +113,7 @@ func (c *Catalog) Logger() *supportlog.Entry { return c.logger } func (c *Catalog) Layout() geometry.Layout { return c.layout } -func (c *Catalog) TxHashIndexLayout() geometry.TxHashIndexLayout { return c.txhashIndex } +func (c *Catalog) TxHashIndexLayout() geometry.TxHashIndexLayout { return c.layout.TxHashIndex() } // --------------------------------------------------------------------------- // Typed artifact-state accessors. @@ -244,7 +241,7 @@ func (c *Catalog) FrozenIndexCoversRange(w geometry.TxHashIndexID, lo, hi chunk. // window is the only one that can cover it — the degenerate single-chunk case of // FrozenIndexCoversRange. func (c *Catalog) FrozenIndexCovers(ch chunk.ID) (bool, error) { - return c.FrozenIndexCoversRange(c.txhashIndex.TxHashIndexID(ch), ch, ch) + return c.FrozenIndexCoversRange(c.TxHashIndexLayout().TxHashIndexID(ch), ch, ch) } // --------------------------------------------------------------------------- diff --git a/cmd/stellar-rpc/internal/rpcv2/catalog/catalog_protocol.go b/cmd/stellar-rpc/internal/rpcv2/catalog/catalog_protocol.go index 95e390073..97ba68438 100644 --- a/cmd/stellar-rpc/internal/rpcv2/catalog/catalog_protocol.go +++ b/cmd/stellar-rpc/internal/rpcv2/catalog/catalog_protocol.go @@ -115,7 +115,7 @@ func (c *Catalog) CommitTxHashIndex(cov geometry.TxHashIndexCoverage) error { } var txhashKeys []string - if c.txhashIndex.IsTerminalCoverage(cov) { + if c.TxHashIndexLayout().IsTerminalCoverage(cov) { txhashKeys, err = c.txhashIndexChunkKeysPresent(cov.Lo, cov.Hi) if err != nil { return err diff --git a/cmd/stellar-rpc/internal/rpcv2/catalog/catalog_protocol_test.go b/cmd/stellar-rpc/internal/rpcv2/catalog/catalog_protocol_test.go index 3577ef595..82dffe8ef 100644 --- a/cmd/stellar-rpc/internal/rpcv2/catalog/catalog_protocol_test.go +++ b/cmd/stellar-rpc/internal/rpcv2/catalog/catalog_protocol_test.go @@ -80,7 +80,7 @@ func TestCommitIndexTerminalDemotesTxhashKeys(t *testing.T) { // Terminal build covers the whole index [0,999] => hi == last chunk. cov, err := cat.MarkTxHashIndexFreezing(0, 0, 999) require.NoError(t, err) - require.True(t, cat.txhashIndex.IsTerminalCoverage(cov)) + require.True(t, cat.TxHashIndexLayout().IsTerminalCoverage(cov)) require.NoError(t, cat.CommitTxHashIndex(cov)) // Every present txhash key in the index demoted to "pruning". @@ -110,7 +110,7 @@ func TestCommitIndexNonTerminalLeavesTxhashKeys(t *testing.T) { // Non-terminal: hi (5) < index's last chunk (999). cov, err := cat.MarkTxHashIndexFreezing(0, 0, 5) require.NoError(t, err) - require.False(t, cat.txhashIndex.IsTerminalCoverage(cov)) + require.False(t, cat.TxHashIndexLayout().IsTerminalCoverage(cov)) require.NoError(t, cat.CommitTxHashIndex(cov)) // txhash key NOT demoted — the index is still filling. @@ -139,7 +139,7 @@ func TestCommitIndexTerminalDemotesOnlyCoverageRange(t *testing.T) { // A TERMINAL coverage starting after the index's first chunk: [5500,5999]. cov, err := cat.MarkTxHashIndexFreezing(5, 5500, 5999) require.NoError(t, err) - require.True(t, cat.txhashIndex.IsTerminalCoverage(cov)) + require.True(t, cat.TxHashIndexLayout().IsTerminalCoverage(cov)) require.NoError(t, cat.CommitTxHashIndex(cov)) // .bin inputs inside [Lo,Hi] are demoted... diff --git a/cmd/stellar-rpc/internal/rpcv2/catalog/catalog_sweep.go b/cmd/stellar-rpc/internal/rpcv2/catalog/catalog_sweep.go index 0487aa530..27cb8f5b6 100644 --- a/cmd/stellar-rpc/internal/rpcv2/catalog/catalog_sweep.go +++ b/cmd/stellar-rpc/internal/rpcv2/catalog/catalog_sweep.go @@ -133,7 +133,7 @@ func (c *Catalog) DestroyTxHashIndexKey(cov geometry.TxHashIndexCoverage) error if err := durable.DeleteFileIfExists(path); err != nil { return err } - dir := c.layout.TxHashIndexDir(cov.Index) + dir := c.layout.TxHashIndexDir(cov) if err := durable.FsyncDir(dir); err != nil { // unlink durable BEFORE key delete return err } diff --git a/cmd/stellar-rpc/internal/rpcv2/catalog/catalog_sweep_test.go b/cmd/stellar-rpc/internal/rpcv2/catalog/catalog_sweep_test.go index 092c960ab..815166ef0 100644 --- a/cmd/stellar-rpc/internal/rpcv2/catalog/catalog_sweep_test.go +++ b/cmd/stellar-rpc/internal/rpcv2/catalog/catalog_sweep_test.go @@ -3,6 +3,7 @@ package catalog import ( "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/require" @@ -89,6 +90,7 @@ func TestSweepIndexKey(t *testing.T) { cov, err := cat.MarkTxHashIndexFreezing(5, 5100, 5349) require.NoError(t, err) idxPath := cat.layout.TxHashIndexFilePath(cov) + require.True(t, strings.HasPrefix(idxPath, cat.layout.HotRoot()), "a window still filling lives with the hot tier") writeArtifact(t, idxPath) require.NoError(t, cat.CommitTxHashIndex(cov)) diff --git a/cmd/stellar-rpc/internal/rpcv2/catalog/census.go b/cmd/stellar-rpc/internal/rpcv2/catalog/census.go index 2cbcb3467..de18edbc4 100644 --- a/cmd/stellar-rpc/internal/rpcv2/catalog/census.go +++ b/cmd/stellar-rpc/internal/rpcv2/catalog/census.go @@ -96,8 +96,8 @@ func (c *Catalog) censusArtifactEntry(key, value string) (string, bool) { // The builder only ever covers chunks of the key's own index, so a // cross-window coverage is something no binary wrote. Left accepted, // a corrupt frozen one would suppress legitimate index rebuilds. - if c.txhashIndex.TxHashIndexID(cov.Lo) != cov.Index || - c.txhashIndex.TxHashIndexID(cov.Hi) != cov.Index { + if c.TxHashIndexLayout().TxHashIndexID(cov.Lo) != cov.Index || + c.TxHashIndexLayout().TxHashIndexID(cov.Hi) != cov.Index { return "index coverage endpoints outside the key's own index", false } if !geometry.IsKnownState(geometry.State(value)) { diff --git a/cmd/stellar-rpc/internal/rpcv2/catalog/crashsafety_test.go b/cmd/stellar-rpc/internal/rpcv2/catalog/crashsafety_test.go index 581f48303..e23e3cb28 100644 --- a/cmd/stellar-rpc/internal/rpcv2/catalog/crashsafety_test.go +++ b/cmd/stellar-rpc/internal/rpcv2/catalog/crashsafety_test.go @@ -46,6 +46,13 @@ func TestCrashSafety_FileWrittenKeyNotFlipped(t *testing.T) { require.NoError(t, durable.BarrierNewFile(idxPath)) // <-- crash here: no CommitTxHashIndex. + // The same for a terminal coverage, whose file is under the cold root. + terminal, err := cat.MarkTxHashIndexFreezing(6, 6000, 6999) + require.NoError(t, err) + terminalPath := cat.layout.TxHashIndexFilePath(terminal) + writeArtifact(t, terminalPath) + require.NoError(t, durable.BarrierNewFile(terminalPath)) + // INV-3 (disk -> meta): every file on disk has its key. assertEveryFileHasKey(t, cat, root) diff --git a/cmd/stellar-rpc/internal/rpcv2/catalog/helpers_test.go b/cmd/stellar-rpc/internal/rpcv2/catalog/helpers_test.go index e16553bcb..05527e8ab 100644 --- a/cmd/stellar-rpc/internal/rpcv2/catalog/helpers_test.go +++ b/cmd/stellar-rpc/internal/rpcv2/catalog/helpers_test.go @@ -33,10 +33,7 @@ func testCatalog(t *testing.T) (*Catalog, string) { metaDir := t.TempDir() artifactRoot := t.TempDir() - idxLayout, err := geometry.NewTxHashIndexLayout(geometry.ChunksPerTxhashIndex) - require.NoError(t, err) - - cat, err := Open(filepath.Join(metaDir, "rocksdb"), geometry.NewLayout(artifactRoot), idxLayout, silentLogger()) + cat, err := Open(filepath.Join(metaDir, "rocksdb"), geometry.NewLayout(artifactRoot), silentLogger()) require.NoError(t, err) t.Cleanup(func() { _ = cat.Close() }) @@ -76,12 +73,10 @@ func assertEveryFileHasKey(t *testing.T, cat *Catalog, root string) { func keyForArtifactFile(t *testing.T, cat *Catalog, path string) (string, bool) { t.Helper() - // Index file: txhash/index/{w}/{lo}-{hi}.idx - dir := filepath.Dir(path) + // Index file: {lo}-{hi}.idx, under the cold index root once terminal and + // under the hot root before; lo names the window either way. base := filepath.Base(path) if filepath.Ext(base) == ".idx" { - w, errW := geometry.ParsePadded(filepath.Base(dir)) - require.NoError(t, errW) name := strings.TrimSuffix(base, ".idx") loStr, hiStr, found := strings.Cut(name, "-") require.True(t, found, "bad idx name %q", base) @@ -89,7 +84,11 @@ func keyForArtifactFile(t *testing.T, cat *Catalog, path string) (string, bool) require.NoError(t, errLo) hi, errHi := geometry.ParsePadded(hiStr) require.NoError(t, errHi) - return geometry.TxHashIndexKey(geometry.TxHashIndexID(w), chunk.ID(lo), chunk.ID(hi)), true + cov := geometry.TxHashIndexCoverage{ + Index: cat.TxHashIndexLayout().TxHashIndexID(chunk.ID(lo)), Lo: chunk.ID(lo), Hi: chunk.ID(hi), + } + require.Equal(t, cat.layout.TxHashIndexFilePath(cov), path, "an .idx outside its coverage's directory") + return geometry.TxHashIndexKey(cov.Index, cov.Lo, cov.Hi), true } // Per-chunk files: identify by reconstructing each kind's path for the diff --git a/cmd/stellar-rpc/internal/rpcv2/catalog/kv_test.go b/cmd/stellar-rpc/internal/rpcv2/catalog/kv_test.go index 78fdcffa4..e746af6e9 100644 --- a/cmd/stellar-rpc/internal/rpcv2/catalog/kv_test.go +++ b/cmd/stellar-rpc/internal/rpcv2/catalog/kv_test.go @@ -23,9 +23,7 @@ import ( // cases); the artifact layout is irrelevant to the KV tests. func openKVAt(t *testing.T, path string) (*Catalog, error) { t.Helper() - idxLayout, err := geometry.NewTxHashIndexLayout(geometry.ChunksPerTxhashIndex) - require.NoError(t, err) - return Open(path, geometry.NewLayout(t.TempDir()), idxLayout, silentLogger()) + return Open(path, geometry.NewLayout(t.TempDir()), silentLogger()) } func openTestKV(t *testing.T) *Catalog { @@ -54,13 +52,10 @@ func kvGetMiss(t *testing.T, c *Catalog, key string) { } func TestOpen_ValidatesInputs(t *testing.T) { - idxLayout, err := geometry.NewTxHashIndexLayout(geometry.ChunksPerTxhashIndex) - require.NoError(t, err) - - _, err = Open("", geometry.NewLayout(t.TempDir()), idxLayout, silentLogger()) + _, err := Open("", geometry.NewLayout(t.TempDir()), silentLogger()) require.ErrorIs(t, err, rocksdb.ErrInvalidConfig) - _, err = Open(t.TempDir(), geometry.NewLayout(t.TempDir()), idxLayout, nil) + _, err = Open(t.TempDir(), geometry.NewLayout(t.TempDir()), nil) require.ErrorIs(t, err, rocksdb.ErrInvalidConfig) } diff --git a/cmd/stellar-rpc/internal/rpcv2/config/config_test.go b/cmd/stellar-rpc/internal/rpcv2/config/config_test.go index 2dad3d07f..11dec28a4 100644 --- a/cmd/stellar-rpc/internal/rpcv2/config/config_test.go +++ b/cmd/stellar-rpc/internal/rpcv2/config/config_test.go @@ -500,8 +500,8 @@ func TestRoots_AllDistinct(t *testing.T) { cfg, err := ParseConfig([]byte(minimalValidConfig)) require.NoError(t, err) roots := cfg.ResolvePaths().Roots() - // Meta store + five immutable trees + hot storage = seven roots. The - // events pack and its index are separate trees so a deployment can put + // Meta store + five immutable trees + hot storage = seven roots. + // The events pack and its index are separate trees so a deployment can put // the randomly-probed index on faster storage than the bulk pack. require.Len(t, roots, 7) assert.NotContains(t, roots, "/data", "the data dir parent is not itself a root") diff --git a/cmd/stellar-rpc/internal/rpcv2/config/rpc-v2-sample-config.toml b/cmd/stellar-rpc/internal/rpcv2/config/rpc-v2-sample-config.toml index d0a0d6e17..8a7cb37d2 100644 --- a/cmd/stellar-rpc/internal/rpcv2/config/rpc-v2-sample-config.toml +++ b/cmd/stellar-rpc/internal/rpcv2/config/rpc-v2-sample-config.toml @@ -33,16 +33,19 @@ default_data_dir = "/var/stellar/rpc-v2" # Per-store paths, for putting stores on different disks. Relative paths # resolve against the working directory, and two stores cannot share a path. -# events and events_index move independently. The indexes are small and read -# randomly, so putting events_index on fast local disk speeds up filtered +# With a local NVMe disk and a slower durable volume, put hot and txhash_raw +# (and [ingestion] captive_core_storage_path) on the NVMe disk and keep the +# catalog on the durable volume. The other stores can sit on the slower +# volume. events and events_index move independently: the indexes are small +# and read randomly, so putting events_index on fast disk speeds up filtered # event queries over wide ranges. #catalog = "" # {default_data_dir}/catalog/rocksdb (catalog RocksDB) #ledgers = "" # {default_data_dir}/ledgers (ledger packs) #events = "" # {default_data_dir}/events/data (event packs) #events_index = "" # {default_data_dir}/events/index (event indexes) #txhash_raw = "" # {default_data_dir}/txhash/raw (unindexed tx hashes) -#txhash_index = "" # {default_data_dir}/txhash/index (txhash indexes) -#hot = "" # {default_data_dir}/hot (recent chunks) +#txhash_index = "" # {default_data_dir}/txhash/index (terminal txhash indexes) +#hot = "" # {default_data_dir}/hot (recent chunks, the txhash index still being rebuilt, build scratch) # ----------------------------------------------------------------------------- # [retention]: how much history this daemon keeps diff --git a/cmd/stellar-rpc/internal/rpcv2/daemon.go b/cmd/stellar-rpc/internal/rpcv2/daemon.go index e32a5bcc6..e828249fa 100644 --- a/cmd/stellar-rpc/internal/rpcv2/daemon.go +++ b/cmd/stellar-rpc/internal/rpcv2/daemon.go @@ -346,7 +346,7 @@ func openCatalog(paths config.Paths, opts daemonOptions, logger *supportlog.Entr if err != nil { return nil, err } - cat, err := catalog.Open(paths.Catalog, config.NewLayoutFromPaths(paths), txLayout, logger) + cat, err := catalog.Open(paths.Catalog, config.NewLayoutFromPaths(paths).WithTxHashIndex(txLayout), logger) if err != nil { return nil, fmt.Errorf("open catalog %q: %w", paths.Catalog, err) } diff --git a/cmd/stellar-rpc/internal/rpcv2/daemon_test.go b/cmd/stellar-rpc/internal/rpcv2/daemon_test.go index 015c22de7..c1bf7d230 100644 --- a/cmd/stellar-rpc/internal/rpcv2/daemon_test.go +++ b/cmd/stellar-rpc/internal/rpcv2/daemon_test.go @@ -34,10 +34,7 @@ import ( // read-back assertions (closed via t.Cleanup). func openCatalogAt(t *testing.T, dataDir string) *catalog.Catalog { t.Helper() - txLayout, err := geometry.NewTxHashIndexLayout(geometry.ChunksPerTxhashIndex) - require.NoError(t, err) - cat, err := catalog.Open( - filepath.Join(dataDir, "catalog", "rocksdb"), geometry.NewLayout(dataDir), txLayout, silentLogger()) + cat, err := catalog.Open(filepath.Join(dataDir, "catalog", "rocksdb"), geometry.NewLayout(dataDir), silentLogger()) require.NoError(t, err) t.Cleanup(func() { _ = cat.Close() }) return cat diff --git a/cmd/stellar-rpc/internal/rpcv2/e2e_test.go b/cmd/stellar-rpc/internal/rpcv2/e2e_test.go index 87a692bb6..f8720aebc 100644 --- a/cmd/stellar-rpc/internal/rpcv2/e2e_test.go +++ b/cmd/stellar-rpc/internal/rpcv2/e2e_test.go @@ -528,7 +528,7 @@ func e2eReadCatalog(t *testing.T, dataDir string) (*catalog.Catalog, func()) { paths := config.Config{Storage: config.StorageConfig{DefaultDataDir: dataDir}}.WithDefaults().ResolvePaths() windows, err := geometry.NewTxHashIndexLayout(1) // matches chunksPerTxhashIndex = 1 require.NoError(t, err) - cat, err := catalog.Open(paths.Catalog, config.NewLayoutFromPaths(paths), windows, silentLogger()) + cat, err := catalog.Open(paths.Catalog, config.NewLayoutFromPaths(paths).WithTxHashIndex(windows), silentLogger()) require.NoError(t, err) return cat, func() { _ = cat.Close() } } diff --git a/cmd/stellar-rpc/internal/rpcv2/geometry/keys_test.go b/cmd/stellar-rpc/internal/rpcv2/geometry/keys_test.go index c44b2d9db..19e7dfef9 100644 --- a/cmd/stellar-rpc/internal/rpcv2/geometry/keys_test.go +++ b/cmd/stellar-rpc/internal/rpcv2/geometry/keys_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/rpcv2/chunk" + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/rpcv2/stores/event" ) // --------------------------------------------------------------------------- @@ -57,13 +58,26 @@ func TestKeyToPathBijection(t *testing.T) { "/data/events/index/00005/00005350-index.pack", "/data/events/index/00005/00005350-index.hash", }, l.EventsPaths(5350)) - require.Equal(t, []string{"/data/events/index/00005/00005350-index.runs"}, l.ScratchPaths(5350, KindEvents)) + require.Equal(t, []string{"/data/hot/scratch/00005350-index.runs"}, l.ScratchPaths(5350, KindEvents)) + require.Equal(t, event.ColdDirs{ + Data: "/data/events/data/00005", Index: "/data/events/index/00005", Scratch: "/data/hot/scratch", + }, l.EventsColdDirs(5350)) require.Nil(t, l.ScratchPaths(5350, KindLedgers)) require.Equal(t, "/data/hot/00005350", l.HotChunkPath(5350)) + // A window still being rebuilt lives with the hot tier; its terminal + // index is a cold artifact. cov := TxHashIndexCoverage{Index: 5, Lo: 5100, Hi: 5349} - require.Equal(t, "/data/txhash/index/00000005", l.TxHashIndexDir(cov.Index)) - require.Equal(t, "/data/txhash/index/00000005/00005100-00005349.idx", l.TxHashIndexFilePath(cov)) + require.Equal(t, "/data/hot/txhash-index-00000005", l.TxHashIndexDir(cov)) + require.Equal(t, "/data/hot/txhash-index-00000005/00005100-00005349.idx", l.TxHashIndexFilePath(cov)) + terminal := TxHashIndexCoverage{Index: 5, Lo: 5100, Hi: 5999} + require.Equal(t, "/data/txhash/index/00000005", l.TxHashIndexDir(terminal)) + require.Equal(t, "/data/txhash/index/00000005/00005100-00005999.idx", l.TxHashIndexFilePath(terminal)) + // The width decides what is terminal, so it decides the placement. + w4, err := NewTxHashIndexLayout(4) + require.NoError(t, err) + require.Equal(t, "/data/txhash/index/00000001", + l.WithTxHashIndex(w4).TxHashIndexDir(TxHashIndexCoverage{Index: 1, Lo: 4, Hi: 7})) } func TestParseRejectsMalformed(t *testing.T) { diff --git a/cmd/stellar-rpc/internal/rpcv2/geometry/paths.go b/cmd/stellar-rpc/internal/rpcv2/geometry/paths.go index 739aa0487..caace0524 100644 --- a/cmd/stellar-rpc/internal/rpcv2/geometry/paths.go +++ b/cmd/stellar-rpc/internal/rpcv2/geometry/paths.go @@ -15,15 +15,17 @@ import ( // // {root}/ // ├── catalog/rocksdb/ -// ├── hot/{chunk:08d}/ +// ├── hot/ +// │ ├── {chunk:08d}/ (per-chunk hot RocksDB) +// │ ├── txhash-index-{window:08d}/{lo:08d}-{hi:08d}.idx (the window still being rebuilt) +// │ └── scratch/{chunk:08d}-index.runs/ (while a chunk's events index is built) // ├── ledgers/{bucket:05d}/{chunk:08d}.pack // ├── events/ // │ ├── data/{bucket:05d}/{chunk:08d}-events.pack // │ └── index/{bucket:05d}/{chunk:08d}-index.pack (+ -index.hash) -// │ (+ {chunk:08d}-index.runs/ while the chunk's events index is being built) // └── txhash/ // ├── raw/{bucket:05d}/{chunk:08d}.bin -// └── index/{idx:08d}/{lo:08d}-{hi:08d}.idx +// └── index/{idx:08d}/{lo:08d}-{hi:08d}.idx (terminal windows) // // Each root is independently settable (NewLayoutFromRoots) for the [storage] // path overrides. Bucket ids never appear in meta-store keys. @@ -35,6 +37,7 @@ type Layout struct { eventsIndexRoot string txhashRawRoot string txhashIndexRoot string + txhash TxHashIndexLayout } // NewLayout is the no-override deployment: NewLayoutFromRoots of the per-tree @@ -48,6 +51,7 @@ func NewLayout(root string) Layout { eventsIndexRoot: filepath.Join(root, "events", "index"), txhashRawRoot: filepath.Join(root, "txhash", "raw"), txhashIndexRoot: filepath.Join(root, "txhash", "index"), + txhash: TxHashIndexLayout{cpi: ChunksPerTxhashIndex}, } } @@ -67,13 +71,25 @@ func NewLayoutFromRoots( eventsIndexRoot: eventsIndexRoot, txhashRawRoot: txhashRawRoot, txhashIndexRoot: txhashIndexRoot, + txhash: TxHashIndexLayout{cpi: ChunksPerTxhashIndex}, } } +// WithTxHashIndex returns the layout with another tx-hash index width, which +// decides where a window's index lives (TxHashIndexDir). +func (l Layout) WithTxHashIndex(t TxHashIndexLayout) Layout { + l.txhash = t + return l +} + +// TxHashIndex is the tx-hash index arithmetic the layout was built with. +func (l Layout) TxHashIndex() TxHashIndexLayout { return l.txhash } + // CatalogPath is the meta-store RocksDB directory. func (l Layout) CatalogPath() string { return l.catalogRoot } -// HotRoot holds the per-chunk hot RocksDB dirs. +// HotRoot is the local tier: the per-chunk hot RocksDB dirs, the index of +// the window still being rebuilt, and the builds' scratch. func (l Layout) HotRoot() string { return l.hotRoot } // HotChunkPath is a chunk's hot RocksDB dir. @@ -111,11 +127,12 @@ func (l Layout) EventsIndexBucketDir(c chunk.ID) string { return filepath.Join(l.eventsIndexRoot, c.BucketID()) } -// EventsColdDirs is the pair of directories a chunk's events artifacts live -// in. Readers take the pair, so no caller composes it by hand and none can -// pick up one root while missing the other. +// EventsColdDirs are the directories a chunk's events artifacts live in and +// the one its index build spills under. Readers take the struct, so no +// caller composes it by hand and none can pick up one root while missing +// another. func (l Layout) EventsColdDirs(c chunk.ID) event.ColdDirs { - return event.ColdDirs{Data: l.EventsBucketDir(c), Index: l.EventsIndexBucketDir(c)} + return event.ColdDirs{Data: l.EventsBucketDir(c), Index: l.EventsIndexBucketDir(c), Scratch: l.scratchDir()} } // EventsPaths are a chunk's three events cold-segment files, which span the @@ -138,7 +155,7 @@ func (l Layout) ScratchPaths(c chunk.ID, kind Kind) []string { if kind != KindEvents { return nil } - return []string{filepath.Join(l.EventsIndexBucketDir(c), event.IndexRunsDirName(c))} + return []string{filepath.Join(l.scratchDir(), event.IndexRunsDirName(c))} } // TxHashBinPath is a chunk's raw txhash run. Leaf owned by txhash.ColdBinName. @@ -161,19 +178,26 @@ func (l Layout) EventsRoot() string { return l.eventsRoot } // coldDir/ layout. func (l Layout) TxHashRawRoot() string { return l.txhashRawRoot } -// TxHashIndexRoot is the root TxHashIndexDir composes under. +// TxHashIndexRoot is the root terminal indexes live under. func (l Layout) TxHashIndexRoot() string { return l.txhashIndexRoot } -// TxHashIndexDir is one index's directory. -func (l Layout) TxHashIndexDir(w TxHashIndexID) string { - return filepath.Join(l.txhashIndexRoot, w.String()) +// TxHashIndexDir is a coverage's directory. A terminal index is a cold +// artifact under the txhash index root. The index of a window still being +// rebuilt at every boundary lives with the hot tier, one directory under the +// hot root beside the chunk DBs, so each rewrite lands there and not under a +// cold root. +func (l Layout) TxHashIndexDir(cov TxHashIndexCoverage) string { + if l.txhash.IsTerminalCoverage(cov) { + return filepath.Join(l.txhashIndexRoot, cov.Index.String()) + } + return filepath.Join(l.hotRoot, "txhash-index-"+cov.Index.String()) } // TxHashIndexFilePath derives the .idx name from a coverage: lo-hi names the range it // covers. func (l Layout) TxHashIndexFilePath(cov TxHashIndexCoverage) string { name := cov.Lo.String() + "-" + cov.Hi.String() + ".idx" - return filepath.Join(l.TxHashIndexDir(cov.Index), name) + return filepath.Join(l.TxHashIndexDir(cov), name) } // ArtifactPaths is the single (chunk, kind)->files map, so the sweep and the @@ -190,3 +214,6 @@ func (l Layout) ArtifactPaths(c chunk.ID, kind Kind) []string { return nil } } + +// scratchDir is where builds keep what they remove before their key flips. +func (l Layout) scratchDir() string { return filepath.Join(l.hotRoot, "scratch") } diff --git a/cmd/stellar-rpc/internal/rpcv2/ingest/driver.go b/cmd/stellar-rpc/internal/rpcv2/ingest/driver.go index 2ad4845ca..6e507bee5 100644 --- a/cmd/stellar-rpc/internal/rpcv2/ingest/driver.go +++ b/cmd/stellar-rpc/internal/rpcv2/ingest/driver.go @@ -51,7 +51,7 @@ func drain(ledgers iter.Seq2[[]byte, error], chunkID chunk.ID, cc *coldChunk) er // caller from geometry.Layout so the ingesters write exactly where the freeze // barrier and the sweeps resolve — the path formula lives in Layout alone, never // re-derived here. LedgerPack and TxhashBin are the chunk's full file paths; -// Events is its pair of events bucket dirs. An empty field for an enabled type is a +// Events is its events directories. An empty field for an enabled type is a // config error. type ColdDirs struct { LedgerPack string diff --git a/cmd/stellar-rpc/internal/rpcv2/ingest/events.go b/cmd/stellar-rpc/internal/rpcv2/ingest/events.go index ad662d2fa..79b339759 100644 --- a/cmd/stellar-rpc/internal/rpcv2/ingest/events.go +++ b/cmd/stellar-rpc/internal/rpcv2/ingest/events.go @@ -39,7 +39,7 @@ type eventsCold struct { // newEventsCold opens a per-chunk events cold writer over dirs — // the caller's geometry.Layout.EventsColdDirs(chunkID), so the write path is -// Layout's single derivation of both roots. The writer opts into the batch tuning +// Layout's single derivation of the roots. The writer opts into the batch tuning // (coldEncoderConcurrency/coldBytesPerSync): WriteColdChunk, the sole // production caller, is always a batch freeze/backfill. func newEventsCold( @@ -62,7 +62,7 @@ func newEventsCold( return &eventsCold{ chunkID: chunkID, writer: w, - index: event.NewColdIndexBuilder(chunkID, dirs.Index, secret), + index: event.NewColdIndexBuilder(chunkID, dirs, secret), offsets: event.NewLedgerOffsets(chunkID.FirstLedger()), metrics: newColdMetrics(sink, dataTypeEvents), }, nil diff --git a/cmd/stellar-rpc/internal/rpcv2/ingest/ingest_test.go b/cmd/stellar-rpc/internal/rpcv2/ingest/ingest_test.go index ba370d312..7e1cd1952 100644 --- a/cmd/stellar-rpc/internal/rpcv2/ingest/ingest_test.go +++ b/cmd/stellar-rpc/internal/rpcv2/ingest/ingest_test.go @@ -254,7 +254,7 @@ func packPath(ledgersRoot string, c chunk.ID) string { // sameDirs points both events roots at one directory — the shape a test that // only cares about the files wants, and one the store supports even though a // deployment must keep the two roots apart. -func sameDirs(dir string) event.ColdDirs { return event.ColdDirs{Data: dir, Index: dir} } +func sameDirs(dir string) event.ColdDirs { return event.ColdDirs{Data: dir, Index: dir, Scratch: dir} } // coldDirsAt resolves chunk c's cold-artifact paths under one dir's per-type // roots — mirroring what geometry.Layout derives in production, so the readback @@ -1920,8 +1920,9 @@ func TestEventsCold_CreatesBothRootsOnFreeze(t *testing.T) { // Two roots, neither of which exists, as an operator gets on a fresh // deployment that puts the index on separate storage. dirs := event.ColdDirs{ - Data: filepath.Join(root, "events", "data", chunkID.BucketID()), - Index: filepath.Join(root, "events", "index", chunkID.BucketID()), + Data: filepath.Join(root, "events", "data", chunkID.BucketID()), + Index: filepath.Join(root, "events", "index", chunkID.BucketID()), + Scratch: filepath.Join(root, "hot", "scratch"), } require.NoDirExists(t, dirs.Data) require.NoDirExists(t, dirs.Index) diff --git a/cmd/stellar-rpc/internal/rpcv2/rpcv2test/rpcv2test.go b/cmd/stellar-rpc/internal/rpcv2/rpcv2test/rpcv2test.go index d4ed372e6..9f61c9924 100644 --- a/cmd/stellar-rpc/internal/rpcv2/rpcv2test/rpcv2test.go +++ b/cmd/stellar-rpc/internal/rpcv2/rpcv2test/rpcv2test.go @@ -60,7 +60,7 @@ func OpenTestCatalogWith(t testing.TB, cpi uint32, logger *supportlog.Entry) (*c idxLayout, err := geometry.NewTxHashIndexLayout(cpi) require.NoError(t, err) cat, err := catalog.Open( - filepath.Join(t.TempDir(), "rocksdb"), geometry.NewLayout(artifactRoot), idxLayout, logger) + filepath.Join(t.TempDir(), "rocksdb"), geometry.NewLayout(artifactRoot).WithTxHashIndex(idxLayout), logger) require.NoError(t, err) t.Cleanup(func() { _ = cat.Close() }) return cat, artifactRoot diff --git a/cmd/stellar-rpc/internal/rpcv2/stores/event/cold_dirs.go b/cmd/stellar-rpc/internal/rpcv2/stores/event/cold_dirs.go index 0abaca834..5a5d8d6ec 100644 --- a/cmd/stellar-rpc/internal/rpcv2/stores/event/cold_dirs.go +++ b/cmd/stellar-rpc/internal/rpcv2/stores/event/cold_dirs.go @@ -1,14 +1,15 @@ package event -// ColdDirs are the two bucket directories a chunk's cold events artifacts -// live in, composed by geometry.Layout. They are separate roots because the -// pack is streamed sequentially and holds almost all of the bytes while +// ColdDirs are the directories a chunk's cold events artifacts live in, +// composed by geometry.Layout. Data and Index are separate roots because +// the pack is streamed sequentially and holds almost all of the bytes while // index.pack is probed at random, so a deployment can give the probes the // IOPS they need without moving the pack with them. // // ValidateRoots rejects two storage roots resolving to one path, so in a -// deployment these always differ. Tests may point both at one directory. +// deployment these always differ. Tests may point all three at one directory. type ColdDirs struct { - Data string - Index string + Data string + Index string + Scratch string // where a build keeps what it removes before the chunk freezes } diff --git a/cmd/stellar-rpc/internal/rpcv2/stores/event/cold_format.go b/cmd/stellar-rpc/internal/rpcv2/stores/event/cold_format.go index ba80737ad..34977f064 100644 --- a/cmd/stellar-rpc/internal/rpcv2/stores/event/cold_format.go +++ b/cmd/stellar-rpc/internal/rpcv2/stores/event/cold_format.go @@ -50,7 +50,7 @@ import ( // Layout: {events_root}/{bucketID:05d}/{chunkID:08d}-events.pack and // {events_index_root}/{bucketID:05d}/{chunkID:08d}-index.pack (and // -index.hash). The two roots are distinct, so a chunk's three files -// are not in one directory. ColdDirs carries the pair. +// are not in one directory. ColdDirs carries the directories. // // Bucket path composition is the orchestrator's job — this package // takes bucket directories and composes the per-chunk filename via diff --git a/cmd/stellar-rpc/internal/rpcv2/stores/event/cold_index.go b/cmd/stellar-rpc/internal/rpcv2/stores/event/cold_index.go index 3082418df..241a1bffe 100644 --- a/cmd/stellar-rpc/internal/rpcv2/stores/event/cold_index.go +++ b/cmd/stellar-rpc/internal/rpcv2/stores/event/cold_index.go @@ -37,9 +37,9 @@ func ColdIndexSecret(catalogSecret []byte, chunkID chunk.ID) [stores.SecretLen]b // ColdIndexBuilder builds a chunk's index.hash and index.pack from its // postings with bounded memory. One slab of postings is in memory; it is -// written as a run in a directory beside the index files when the next -// slab starts or the buffer fills, and Write merges the runs. Both cold -// backfill and the live-chunk freeze feed it single-threaded, re-deriving +// written as a run under the scratch directory when the next slab starts +// or the buffer fills, and Write merges the runs. Both cold backfill and +// the live-chunk freeze feed it single-threaded, re-deriving // terms from raw LCMs with TermsForBytes. The directory is the chunk's, so // one builder works on a chunk at a time, which the backfill plan // guarantees: it builds a chunk once per pass. @@ -55,14 +55,14 @@ type ColdIndexBuilder struct { } // NewColdIndexBuilder returns a builder that writes chunkID's index into -// bucketDir, the chunk's INDEX bucket directory (ColdDirs.Index), which must -// exist. secret is the chunk's deterministic routing secret (ColdIndexSecret). -func NewColdIndexBuilder(chunkID chunk.ID, bucketDir string, secret [stores.SecretLen]byte) *ColdIndexBuilder { +// dirs.Index, which must exist, with its runs under dirs.Scratch. secret is +// the chunk's deterministic routing secret (ColdIndexSecret). +func NewColdIndexBuilder(chunkID chunk.ID, dirs ColdDirs, secret [stores.SecretLen]byte) *ColdIndexBuilder { return &ColdIndexBuilder{ chunkID: chunkID, - dir: bucketDir, + dir: dirs.Index, secret: secret, - runs: coldRuns{dir: filepath.Join(bucketDir, IndexRunsDirName(chunkID))}, + runs: coldRuns{dir: filepath.Join(dirs.Scratch, IndexRunsDirName(chunkID))}, live: newHotSlab(0), } } diff --git a/cmd/stellar-rpc/internal/rpcv2/stores/event/cold_index_test.go b/cmd/stellar-rpc/internal/rpcv2/stores/event/cold_index_test.go index c8c78f14e..afd884f40 100644 --- a/cmd/stellar-rpc/internal/rpcv2/stores/event/cold_index_test.go +++ b/cmd/stellar-rpc/internal/rpcv2/stores/event/cold_index_test.go @@ -47,7 +47,7 @@ const indexTestChunkID = chunk.ID(0) func writeColdIndex( ctx context.Context, chunkID chunk.ID, bitmaps Bitmaps, dir string, secret [stores.SecretLen]byte, ) error { - b := NewColdIndexBuilder(chunkID, dir, secret) + b := NewColdIndexBuilder(chunkID, ColdDirs{Data: dir, Index: dir, Scratch: dir}, secret) type cursor struct { key TermKey next uint32 @@ -424,18 +424,19 @@ func TestIndexPack_IndexCostsAtMostFourBytesPerRecord(t *testing.T) { } // A build over several slabs, with a term whose bitmap is split, checked -// through the cold reader against the postings fed in. The runs live -// beside the index files only while the build runs. +// through the cold reader against the postings fed in. The runs live under +// scratch only while the build runs. func TestColdIndexBuilder_BuildsAcrossSlabs(t *testing.T) { const total = 9*hotSlabEvents + 100 dir, _ := buildColdFixture(t, indexTestChunkID, 1, 1) - runsDir := filepath.Join(dir, IndexRunsDirName(indexTestChunkID)) + scratch := t.TempDir() + runsDir := filepath.Join(scratch, IndexRunsDirName(indexTestChunkID)) // Leftovers of an attempt that never finished are ignored and go with // the directory. require.NoError(t, os.MkdirAll(runsDir, 0o755)) require.NoError(t, os.WriteFile(filepath.Join(runsDir, "stale"), []byte("x"), 0o600)) - b := NewColdIndexBuilder(indexTestChunkID, dir, testIndexSecret) + b := NewColdIndexBuilder(indexTestChunkID, ColdDirs{Data: dir, Index: dir, Scratch: scratch}, testIndexSecret) key := func(name string) TermKey { return ComputeTermKey([]byte(name), FieldContractID) } want := NewBitmaps() for id := range uint32(total) { @@ -458,6 +459,7 @@ func TestColdIndexBuilder_BuildsAcrossSlabs(t *testing.T) { } require.FileExists(t, filepath.Join(runsDir, "00000")) require.FileExists(t, filepath.Join(runsDir, "stale")) + require.NoDirExists(t, filepath.Join(dir, IndexRunsDirName(indexTestChunkID))) require.NoError(t, b.Write(context.Background())) require.NoDirExists(t, runsDir) require.ErrorContains(t, b.Write(context.Background()), "already written") @@ -502,7 +504,7 @@ func binaryID(id uint32) []byte { // slab. func TestColdIndexBuilder_SpillsAFullBuffer(t *testing.T) { dir, _ := buildColdFixture(t, indexTestChunkID, 1, 1) - b := NewColdIndexBuilder(indexTestChunkID, dir, testIndexSecret) + b := NewColdIndexBuilder(indexTestChunkID, ColdDirs{Data: dir, Index: dir, Scratch: dir}, testIndexSecret) pool := make([]TermKey, 100) for i := range pool { pool[i] = ComputeTermKey(fmt.Appendf(nil, "pool-%d", i), FieldContractID) @@ -541,7 +543,7 @@ func TestColdIndexBuilder_RemovesRuns(t *testing.T) { build := func(t *testing.T) (*ColdIndexBuilder, string, string) { t.Helper() dir := t.TempDir() - b := NewColdIndexBuilder(chunk.ID(0), dir, testIndexSecret) + b := NewColdIndexBuilder(chunk.ID(0), ColdDirs{Data: dir, Index: dir, Scratch: dir}, testIndexSecret) for id := range uint32(hotSlabEvents + 1) { require.NoError(t, b.Add(id, []TermKey{{1}})) } diff --git a/design-docs/full-history-streaming-workflow.md b/design-docs/full-history-streaming-workflow.md index d6f454c9a..c2651f988 100644 --- a/design-docs/full-history-streaming-workflow.md +++ b/design-docs/full-history-streaming-workflow.md @@ -103,8 +103,8 @@ Every TOML leaf is also settable from the command line: | `events` | `{default_data_dir}/events/data` | events `.pack` segments | | `events_index` | `{default_data_dir}/events/index` | events `index.pack` + `index.hash` | | `txhash_raw` | `{default_data_dir}/txhash/raw` | transient `.bin` files | -| `txhash_index` | `{default_data_dir}/txhash/index` | per-window `.idx` | -| `hot` | `{default_data_dir}/hot` | per-chunk hot RocksDB databases | +| `txhash_index` | `{default_data_dir}/txhash/index` | the `.idx` of each terminal window | +| `hot` | `{default_data_dir}/hot` | per-chunk hot RocksDB databases, the `.idx` of the window still being rebuilt, and build scratch | **[backfill]** @@ -181,15 +181,17 @@ Chunk-level files group into buckets of 1,000 chunks (`bucket_id = chunk_id / 10 ``` {default_data_dir}/ ├── catalog/rocksdb/ ← catalog (WAL always on) -├── hot/{chunk:08d}/ ← per-chunk hot RocksDB (transient) +├── hot/ +│ ├── {chunk:08d}/ ← per-chunk hot RocksDB (transient) +│ ├── txhash-index-{window:08d}/{lo:08d}-{hi:08d}.idx ← the window still being rebuilt +│ └── scratch/{chunk:08d}-index.runs/ ← while the events key is "freezing": the index build's spilled slabs, removed before the key flips ├── ledgers/{bucket:05d}/{chunk:08d}.pack ├── events/ │ ├── data/{bucket:05d}/{chunk:08d}-events.pack │ └── index/{bucket:05d}/{chunk:08d}-index.pack (+ -index.hash) -│ (+ {chunk:08d}-index.runs/ while the events key is "freezing": the index build's spilled slabs, removed before the key flips) └── txhash/ ├── raw/{bucket:05d}/{chunk:08d}.bin ← transient until window finalization (or retention pruning) - └── index/{window:08d}/{lo:08d}-{hi:08d}.idx ← one frozen file per window, coverage-named + └── index/{window:08d}/{lo:08d}-{hi:08d}.idx ← one frozen file per terminal window, coverage-named ``` ### The chunk hot DB @@ -237,7 +239,7 @@ For the per-chunk keys, `"freezing"` means the immutable file is being written; ### Index keys -An index key `index:{txhash_index:08d}:{lo:08d}:{hi:08d}` names the chunk range `[lo, hi]` that its `.idx` covers, mapping 1:1 to the file `txhash/index/{txhash_index:08d}/{lo:08d}-{hi:08d}.idx`. +An index key `index:{txhash_index:08d}:{lo:08d}:{hi:08d}` names the chunk range `[lo, hi]` that its `.idx` covers, mapping 1:1 to the file `{lo:08d}-{hi:08d}.idx`: under `txhash/index/{txhash_index:08d}/` once the coverage is terminal, and under `hot/txhash-index-{txhash_index:08d}/` while the window is still being rebuilt, so the per-boundary rewrite stays with the hot tier. `hi` grows as the window fills: each rebuild folds in the chunks frozen since the last one, advancing `hi` — by one in steady state, by many when catching up. When `hi` reaches the window's last chunk, the window is **complete** and its index is **terminal** — rebuilt again only if retention widening later drops the floor into the window, when backfill re-derives the spent `.bin` inputs from the local packs and rebuilds the index wider. @@ -510,7 +512,7 @@ Startup runs in two steps, both in `startStreaming` below: 1. **Backfill** brings on-disk coverage in line with the retention window, up through the last *complete* chunk at the tip. The partial chunk still forming at the tip is left to hot-DB ingestion: on a restart its ledgers so far are already in the live hot DB, on a first start ingestion fetches them from the resume ledger forward, and either way ingestion completes the chunk as new ledgers arrive. Backfill re-runs if the tip advances mid-pass, and when it returns, the whole in-retention history up to that point is on disk as frozen files — ready to serve. 2. **Serve + ingest** opens the resume chunk's hot DB, starts captive core, serving, the lifecycle goroutine, and the hot-DB ingestion loop. The lifecycle is seeded with the last complete chunk — when one exists; a young network's first run waits for the first boundary — so its first run fires at once and finishes any crash/downtime leftovers concurrently with serving. Reads never wait for it, because a reader only ever resolves a `"ready"` hot DB or a `"frozen"` cold file — never a transient key. -Operational note — **peak disk after long downtime**: pruning runs only in the first run's prune stage, *after* backfill has materialized every newly-in-retention chunk, so a downtime approaching or exceeding the retention window transiently holds up to ~2× the retention footprint (the stale window plus its replacement). A deep backfill also holds the transient `.bin` inputs for every window it is building at once, not just one (the transactions design, §7.4), and, on the events index root, the spilled index runs of every events build in flight (about one index.pack's worth per chunk, `workers` chunks at once). Size volumes accordingly; a disk-full during backfill fails the run before the relieving prune can fire, on every pass. +Operational note — **peak disk after long downtime**: pruning runs only in the first run's prune stage, *after* backfill has materialized every newly-in-retention chunk, so a downtime approaching or exceeding the retention window transiently holds up to ~2× the retention footprint (the stale window plus its replacement). A deep backfill also holds the transient `.bin` inputs for every window it is building at once, not just one (the transactions design, §7.4), and, under the hot root's `scratch/`, the spilled index runs of every events build in flight (about one index.pack's worth per chunk, `workers` chunks at once). Size volumes accordingly; a disk-full during backfill fails the run before the relieving prune can fire, on every pass. The retention floor and resume point are computed by: diff --git a/design-docs/getevents-full-history-design.md b/design-docs/getevents-full-history-design.md index c65570240..a6f6e48f7 100644 --- a/design-docs/getevents-full-history-design.md +++ b/design-docs/getevents-full-history-design.md @@ -191,10 +191,10 @@ The pack and the index are **separate storage roots**. They are read completely differently — the pack is streamed sequentially and holds almost all of the bytes, while `index.pack` is probed at random — so a deployment can put the index on storage with the IOPS to serve those -probes without moving the pack's terabytes with it. The index root also -takes the build's scratch: the runs an index build spills -(`{chunk}-index.runs/`) are written there and read back three times at -finalize. The two roots default to siblings: +probes without moving the pack's terabytes with it. The runs an index +build spills (`{chunk}-index.runs/`) are written under the hot root's +`scratch/` directory and read back three times at finalize. The two roots +default to siblings: ``` events/ @@ -212,7 +212,6 @@ events/ │ ├── 00000000-index.pack │ ├── 00000001-index.hash │ ├── 00000001-index.pack - │ ├── 00000002-index.runs/ (while chunk 2's index is being built) │ ... ├── 00001/ │ ... @@ -395,7 +394,7 @@ When populating cold segments from historical ledger data, the system writes col For each segment (10,000 ledgers): 1. For each ledger: a. Append events to events.pack (record compression is handled internally by the packfile library). - b. Add the event's terms to the in-memory slab; when the next slab starts, write the slab's terms as one sorted run under `{chunk}-index.runs/` beside the index files. + b. Add the event's terms to the in-memory slab; when the next slab starts, write the slab's terms as one sorted run under `{chunk}-index.runs/` in the hot root's `scratch/` directory. c. Update in-memory ledger offset array. 2. At segment completion (10,000 ledgers):