From 09e4651e598ac6114acb18e2615308935c341de0 Mon Sep 17 00:00:00 2001 From: R0ck Date: Sun, 11 Oct 2026 03:25:50 +0100 Subject: [PATCH 1/5] test(store): read resolved_path pubkeys without decoding to pointers Red: resolvedPubkeysFromJSON (a stub over unmarshalResolvedPath + extractResolvedPubkeys) makes 13 allocations for three pubkeys. The equivalence test pins what decoding returns. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KaR4xu8gVykQBkH8yG8m6H --- cmd/server/neighbor_persist.go | 6 +++ cmd/server/resolved_pubkeys_json_test.go | 51 ++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 cmd/server/resolved_pubkeys_json_test.go diff --git a/cmd/server/neighbor_persist.go b/cmd/server/neighbor_persist.go index d73b972cf..560e84af0 100644 --- a/cmd/server/neighbor_persist.go +++ b/cmd/server/neighbor_persist.go @@ -221,3 +221,9 @@ func unmarshalResolvedPath(s string) []*string { } return result } + +// resolvedPubkeysFromJSON returns the non-empty pubkeys of a resolved_path +// JSON array: extractResolvedPubkeys(unmarshalResolvedPath(s)). +func resolvedPubkeysFromJSON(s string) []string { + return extractResolvedPubkeys(unmarshalResolvedPath(s)) +} diff --git a/cmd/server/resolved_pubkeys_json_test.go b/cmd/server/resolved_pubkeys_json_test.go new file mode 100644 index 000000000..367ad78c0 --- /dev/null +++ b/cmd/server/resolved_pubkeys_json_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "math/rand" + "strings" + "testing" +) + +// The bulk loaders decode each observation's resolved_path (a JSON array of +// pubkeys and nulls, ~420 chars on live data, set on most observations) +// only to collect its non-empty pubkeys. Decoding into []*string allocated +// a string and a pointer per hop plus two slices: about 210MB over one +// startup on a busy week. resolvedPubkeysFromJSON reads the usual array in +// place: one string per pubkey and the result. +func TestResolvedPubkeysFromJSONAllocations(t *testing.T) { + pk := strings.Repeat("ab", 32) + rp := `["` + pk + `",null,"` + strings.Repeat("cd", 32) + `",null,"` + strings.Repeat("ef", 32) + `"]` + // three pubkeys + the result slice + if n := testing.AllocsPerRun(100, func() { _ = resolvedPubkeysFromJSON(rp) }); n > 4 { + t.Fatalf("resolvedPubkeysFromJSON allocates %.0f times for 3 pubkeys, want at most 4", n) + } +} + +// It returns exactly what decoding and extracting would, odd inputs included. +func TestResolvedPubkeysFromJSONMatchesDecoding(t *testing.T) { + cases := []string{ + "", "[]", "null", "[null]", `["aa"]`, `["aa",null,"bb"]`, ` [ "aa" , null ] `, `[""]`, `["",null,""]`, + `["aa",]`, `["aa"`, `["aa"]x`, `[1]`, `["aa",1]`, `[nul]`, `[nulll]`, `["a\"b"]`, `["é"]`, + `[["aa"]]`, `{"a":1}`, "[\"a\x01\"]", `["aa" "bb"]`, `[,]`, `[null,]`, "[\"\xff\"]", `[true]`, + } + rng := rand.New(rand.NewSource(2)) + alphabet := []string{`[`, `]`, `"`, `,`, ` `, `a`, `1`, `\`, `null`, `nul`, `{`, `:`} + for i := 0; i < 20000; i++ { + var b strings.Builder + for j := rng.Intn(12); j >= 0; j-- { + b.WriteString(alphabet[rng.Intn(len(alphabet))]) + } + cases = append(cases, b.String()) + } + for _, s := range cases { + got, want := resolvedPubkeysFromJSON(s), extractResolvedPubkeys(unmarshalResolvedPath(s)) + if len(got) != len(want) { + t.Fatalf("%q: got %q, want %q", s, got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("%q: got %q, want %q", s, got, want) + } + } + } +} From 7c2d78de125ec2f855968b4b85f53f97b05bdb31 Mon Sep 17 00:00:00 2001 From: R0ck Date: Sun, 11 Oct 2026 03:30:25 +0100 Subject: [PATCH 2/5] perf(store): read resolved_path pubkeys in place, one copy each Load, loadChunk and scanAndMergeChunk decode each observation's resolved_path (a JSON array of pubkeys and nulls, ~420 chars, on most observations of live data) only to collect its non-empty pubkeys. The decode into []*string allocated a string and a pointer per hop and two slices. resolvedPubkeysFromJSON reads the usual array of escape-free strings and nulls in place and anything else falls back to decoding, so it returns exactly extractResolvedPubkeys(unmarshalResolvedPath(s)) (pinned on 20k random inputs plus edge cases). Relay pubkeys repeat across nearly every observation, so a bounded table holds one copy of each: a pubkey seen before is found by substring without copying, and the indexes it feeds share that copy. On a copy of a busy week: allocations here over startup 236MB -> 61MB, server CPU over the run 175.6s -> 170.6s. Three known pubkeys: 13 -> 1 allocations. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KaR4xu8gVykQBkH8yG8m6H --- cmd/server/chunked_load.go | 3 +- cmd/server/neighbor_persist.go | 120 ++++++++++++++++++++++- cmd/server/resolved_pubkeys_json_test.go | 9 +- cmd/server/store.go | 6 +- 4 files changed, 127 insertions(+), 11 deletions(-) diff --git a/cmd/server/chunked_load.go b/cmd/server/chunked_load.go index 6420d0eca..864c995fb 100644 --- a/cmd/server/chunked_load.go +++ b/cmd/server/chunked_load.go @@ -629,8 +629,7 @@ func (s *PacketStore) scanAndMergeChunk(rows *sql.Rows, relayPM *prefixMap, cold rpStr := nullStrVal(resolvedPathStr) if rpStr != "" { - rp := unmarshalResolvedPath(rpStr) - pks := extractResolvedPubkeys(rp) + pks := resolvedPubkeysFromJSON(rpStr) s.indexResolvedPathHops(tx, pks, hopsSeen) } else if relayPM != nil && obsPJ != "" && obsPJ != "[]" { // resolved_path is NULL on live (since #1287 relay data is diff --git a/cmd/server/neighbor_persist.go b/cmd/server/neighbor_persist.go index 560e84af0..c73cd70b3 100644 --- a/cmd/server/neighbor_persist.go +++ b/cmd/server/neighbor_persist.go @@ -20,6 +20,7 @@ import ( "encoding/json" "log" "strings" + "sync" "time" ) @@ -223,7 +224,124 @@ func unmarshalResolvedPath(s string) []*string { } // resolvedPubkeysFromJSON returns the non-empty pubkeys of a resolved_path -// JSON array: extractResolvedPubkeys(unmarshalResolvedPath(s)). +// JSON array, exactly extractResolvedPubkeys(unmarshalResolvedPath(s)). The +// bulk loaders call it for most observations; the usual array of plain +// strings and nulls is read in place, one string per pubkey, instead of +// decoding a string and a pointer per hop. Anything else is decoded. func resolvedPubkeysFromJSON(s string) []string { + if s == "" { + return nil + } + if out, ok := scanResolvedPath(s); ok { + return out + } return extractResolvedPubkeys(unmarshalResolvedPath(s)) } + +// resolvedPubkeyStrings holds one copy of each pubkey read from a +// resolved_path. Relay pubkeys repeat across nearly every observation, so +// a pubkey seen before costs no allocation: the lookup by substring does +// not copy, and only a new pubkey is cloned. Bounded so a feed of ever-new +// values cannot grow it without limit; past that, pubkeys are cloned. +var resolvedPubkeyStrings = &pubkeyInterner{m: make(map[string]string)} + +type pubkeyInterner struct { + mu sync.Mutex + m map[string]string +} + +// get returns the held copy of sub, keeping a clone if it is new. sub may +// be a slice of a larger string; the held copy never pins that string. +func (in *pubkeyInterner) get(sub string) string { + in.mu.Lock() + defer in.mu.Unlock() + if v, ok := in.m[sub]; ok { + return v + } + v := strings.Clone(sub) + if len(in.m) < resolvedPubkeyStringsMax { + in.m[v] = v + } + return v +} + +const resolvedPubkeyStringsMax = 1 << 16 + +// scanResolvedPath reads s as a JSON array of escape-free strings and nulls +// with JSON whitespace between tokens. ok is false for anything else. +func scanResolvedPath(s string) (out []string, ok bool) { + i := 0 + ws := func() { + for i < len(s) && (s[i] == ' ' || s[i] == '\t' || s[i] == '\n' || s[i] == '\r') { + i++ + } + } + ws() + if i >= len(s) { + return nil, false + } + if strings.HasPrefix(s[i:], "null") { + // unmarshalResolvedPath gives nil for a bare null. + i += 4 + ws() + return nil, i == len(s) + } + if s[i] != '[' { + return nil, false + } + i++ + ws() + if i < len(s) && s[i] == ']' { + i++ + ws() + return nil, i == len(s) + } + for { + switch { + case i < len(s) && s[i] == '"': + i++ + start := i + for i < len(s) && s[i] != '"' { + if s[i] == '\\' || s[i] < 0x20 || s[i] >= 0x80 { + return nil, false + } + i++ + } + if i >= len(s) { + return nil, false + } + if i > start { + if out == nil { + out = make([]string, 0, 8) + } + out = append(out, resolvedPubkeyStrings.get(s[start:i])) + } + i++ + case strings.HasPrefix(s[i:], "null"): + i += 4 + default: + return nil, false + } + ws() + if i >= len(s) { + return nil, false + } + switch s[i] { + case ',': + i++ + ws() + case ']': + i++ + ws() + if i != len(s) { + return nil, false + } + if out == nil { + out = []string{} + } + return out, true + default: + return nil, false + } + } +} diff --git a/cmd/server/resolved_pubkeys_json_test.go b/cmd/server/resolved_pubkeys_json_test.go index 367ad78c0..10e1b689f 100644 --- a/cmd/server/resolved_pubkeys_json_test.go +++ b/cmd/server/resolved_pubkeys_json_test.go @@ -11,13 +11,14 @@ import ( // only to collect its non-empty pubkeys. Decoding into []*string allocated // a string and a pointer per hop plus two slices: about 210MB over one // startup on a busy week. resolvedPubkeysFromJSON reads the usual array in -// place: one string per pubkey and the result. +// place, and a pubkey it has seen before costs nothing: only the result is +// allocated. func TestResolvedPubkeysFromJSONAllocations(t *testing.T) { pk := strings.Repeat("ab", 32) rp := `["` + pk + `",null,"` + strings.Repeat("cd", 32) + `",null,"` + strings.Repeat("ef", 32) + `"]` - // three pubkeys + the result slice - if n := testing.AllocsPerRun(100, func() { _ = resolvedPubkeysFromJSON(rp) }); n > 4 { - t.Fatalf("resolvedPubkeysFromJSON allocates %.0f times for 3 pubkeys, want at most 4", n) + _ = resolvedPubkeysFromJSON(rp) // first sight keeps one copy of each + if n := testing.AllocsPerRun(100, func() { _ = resolvedPubkeysFromJSON(rp) }); n > 1 { + t.Fatalf("resolvedPubkeysFromJSON allocates %.0f times for 3 known pubkeys, want at most 1 (the result)", n) } } diff --git a/cmd/server/store.go b/cmd/server/store.go index 3c8ffbc18..ec7b73300 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -1074,8 +1074,7 @@ func (s *PacketStore) Load() error { // Decode-window: extract resolved pubkeys for index, don't store on struct. rpStr := nullStrVal(resolvedPathStr) if rpStr != "" { - rp := unmarshalResolvedPath(rpStr) - pks := extractResolvedPubkeys(rp) + pks := resolvedPubkeysFromJSON(rpStr) // Single point of truth — see indexResolvedPathHops doc + #1558. s.indexResolvedPathHops(tx, pks, hopsSeen) } else if relayPM != nil && obsPJ != "" && obsPJ != "[]" { @@ -1419,8 +1418,7 @@ func (s *PacketStore) loadChunk(from, to time.Time) error { // they were heard via — Load() does the equivalent inline. rpStr := nullStrVal(resolvedPathStr) if rpStr != "" { - rp := unmarshalResolvedPath(rpStr) - pks := extractResolvedPubkeys(rp) + pks := resolvedPubkeysFromJSON(rpStr) accumulateDedup(localResolvedPKsByTx, localResolvedSeenByTx, txID, pks) } else if relayPM != nil && obsPJ != "" && obsPJ != "[]" { // resolved_path not persisted — reconstruct relay hops from From ada1eff5df9f948d7aa6e9d753b95bd71d44e773 Mon Sep 17 00:00:00 2001 From: R0ck Date: Sun, 11 Oct 2026 03:51:32 +0100 Subject: [PATCH 3/5] test(store): eviction's resolved pubkey lookup stays cheap Red: resolvedPubkeysForEvictionBatch makes about 51 allocations per evicted packet (3 observations, 4 hops) by decoding each resolved_path into pointers and building a de-dup map per packet. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KaR4xu8gVykQBkH8yG8m6H --- cmd/server/eviction_resolved_pubkeys_test.go | 54 ++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 cmd/server/eviction_resolved_pubkeys_test.go diff --git a/cmd/server/eviction_resolved_pubkeys_test.go b/cmd/server/eviction_resolved_pubkeys_test.go new file mode 100644 index 000000000..96d620ee7 --- /dev/null +++ b/cmd/server/eviction_resolved_pubkeys_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "fmt" + "runtime" + "testing" +) + +func evictionResolvedStore(t *testing.T, n int) (*PacketStore, []int) { + t.Helper() + db := setupTestDB(t) + t.Cleanup(func() { db.Close() }) + pk := func(i int) string { return fmt.Sprintf("%064x", i) } + ids := make([]int, 0, n) + for i := 1; i <= n; i++ { + if _, err := db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, '00', ?, '2026-01-01T00:00:00Z')`, i, fmt.Sprintf("h%015d", i)); err != nil { + t.Fatal(err) + } + for o := 0; o < 3; o++ { + rp := `["` + pk(i%9) + `",null,"` + pk(o+20) + `","` + pk(i%9) + `"]` + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, 1, '["aa","bb","cc","dd"]', 0, ?)`, i, rp); err != nil { + t.Fatal(err) + } + } + ids = append(ids, i) + } + return &PacketStore{db: db}, ids +} + +// Eviction looks up the resolved pubkeys of every packet it drops, once a +// minute in steady state. It decoded each resolved_path into a pointer per +// hop and built a de-dup map per packet: about 130MB of allocations per +// five minutes on a busy week. +func TestResolvedPubkeysForEvictionBatch(t *testing.T) { + s, ids := evictionResolvedStore(t, 300) + got := s.resolvedPubkeysForEvictionBatch(ids) + want := []string{fmt.Sprintf("%064x", 1), fmt.Sprintf("%064x", 20), fmt.Sprintf("%064x", 21), fmt.Sprintf("%064x", 22)} + if fmt.Sprint(got[1]) != fmt.Sprint(want) { + t.Fatalf("tx 1: %v, want %v (first-seen order, de-duplicated)", got[1], want) + } + if len(got) != 300 { + t.Fatalf("%d packets resolved, want 300", len(got)) + } + var a, b runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&a) + _ = s.resolvedPubkeysForEvictionBatch(ids) + runtime.ReadMemStats(&b) + per := float64(b.Mallocs-a.Mallocs) / float64(len(ids)) + t.Logf("%.1f allocations per evicted packet (3 observations, 4 hops each)", per) + if per > 20 { + t.Fatalf("%.1f allocations per evicted packet, limit 20", per) + } +} From d122cfff8deeb677e00e66bc4c3f7c5bc9b71a4c Mon Sep 17 00:00:00 2001 From: R0ck Date: Sun, 11 Oct 2026 03:53:33 +0100 Subject: [PATCH 4/5] perf(store): eviction reads resolved pubkeys in place resolvedPubkeysForEvictionBatch runs for every eviction batch, once a minute in steady state. It scanned each resolved_path into a string, decoded it into a pointer per hop and built a de-dup map per packet. It now scans the column as raw bytes, reads it with resolvedPubkeysFromJSON without copying (nothing keeps a slice of the buffer: new pubkeys are cloned into the shared table, the decoder fallback makes its own strings), and de-duplicates each packet's short list by scanning it. Same keys, same first-seen order. Fixture: 51 -> 15 allocations per evicted packet. A five-minute steady window on a copy of a busy week spent 130MB here. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KaR4xu8gVykQBkH8yG8m6H --- cmd/server/resolved_index.go | 41 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/cmd/server/resolved_index.go b/cmd/server/resolved_index.go index 5b0f05a72..fa0f8f4e4 100644 --- a/cmd/server/resolved_index.go +++ b/cmd/server/resolved_index.go @@ -15,6 +15,7 @@ import ( "log" "sort" "strings" + "unsafe" ) // resolvedPubkeyHash computes a fast 64-bit hash for membership index keying. @@ -516,38 +517,38 @@ func (s *PacketStore) resolvedPubkeysForEvictionBatch(txIDs []int) map[int][]str continue } + var txID int + var rpJSON sql.RawBytes for rows.Next() { - var txID int - var rpJSON sql.NullString if err := rows.Scan(&txID, &rpJSON); err != nil { continue } - if !rpJSON.Valid || rpJSON.String == "" { + if len(rpJSON) == 0 { continue } - rp := unmarshalResolvedPath(rpJSON.String) - for _, p := range rp { - if p != nil && *p != "" { - result[txID] = append(result[txID], *p) + // De-duplicate as we go: a packet has a handful of keys, so a + // scan of its list is cheaper than a map per packet. + pks := result[txID] + // rpJSON is the driver's buffer, valid until the next row: read + // it without copying. Nothing keeps a slice of it (a new pubkey + // is cloned; the decoder fallback makes its own strings). + for _, pk := range resolvedPubkeysFromJSON(unsafe.String(&rpJSON[0], len(rpJSON))) { + dup := false + for _, have := range pks { + if have == pk { + dup = true + break + } + } + if !dup { + pks = append(pks, pk) } } + result[txID] = pks } rows.Close() } - // Deduplicate per-txID - for txID, pks := range result { - seen := make(map[string]bool, len(pks)) - deduped := pks[:0] - for _, pk := range pks { - if !seen[pk] { - seen[pk] = true - deduped = append(deduped, pk) - } - } - result[txID] = deduped - } - return result } From a7f00ad65d680e3d8c539dba797955b9e222d6ec Mon Sep 17 00:00:00 2001 From: R0ck Date: Thu, 15 Oct 2026 10:17:33 +0100 Subject: [PATCH 5/5] perf: intern resolved pubkeys with internStr; scan NullString again Review feedback on #2197: - Eviction scans resolved_path into sql.NullString again. mattn/go-sqlite3 already allocates a string per TEXT column, so RawBytes + unsafe.String only added a copy into database/sql's buffer and an aliasing risk. - pubkeyInterner is folded into internStr, now on master. With the NullString scan, a key returned uninterned past internMax is a slice of that row's own string, not of a reused buffer. - Benchmarks for resolvedPubkeysFromJSON and the eviction batch. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KaR4xu8gVykQBkH8yG8m6H --- cmd/server/eviction_resolved_pubkeys_test.go | 2 +- cmd/server/neighbor_persist.go | 37 ++++---------------- cmd/server/resolved_index.go | 10 ++---- cmd/server/resolved_pubkeys_bench_test.go | 37 ++++++++++++++++++++ 4 files changed, 47 insertions(+), 39 deletions(-) create mode 100644 cmd/server/resolved_pubkeys_bench_test.go diff --git a/cmd/server/eviction_resolved_pubkeys_test.go b/cmd/server/eviction_resolved_pubkeys_test.go index 96d620ee7..078ca351f 100644 --- a/cmd/server/eviction_resolved_pubkeys_test.go +++ b/cmd/server/eviction_resolved_pubkeys_test.go @@ -6,7 +6,7 @@ import ( "testing" ) -func evictionResolvedStore(t *testing.T, n int) (*PacketStore, []int) { +func evictionResolvedStore(t testing.TB, n int) (*PacketStore, []int) { t.Helper() db := setupTestDB(t) t.Cleanup(func() { db.Close() }) diff --git a/cmd/server/neighbor_persist.go b/cmd/server/neighbor_persist.go index c73cd70b3..21eca9aad 100644 --- a/cmd/server/neighbor_persist.go +++ b/cmd/server/neighbor_persist.go @@ -20,7 +20,6 @@ import ( "encoding/json" "log" "strings" - "sync" "time" ) @@ -228,6 +227,11 @@ func unmarshalResolvedPath(s string) []*string { // bulk loaders call it for most observations; the usual array of plain // strings and nulls is read in place, one string per pubkey, instead of // decoding a string and a pointer per hop. Anything else is decoded. +// +// Pubkeys go through internStr: relays repeat across nearly every +// observation, so a pubkey seen before costs no allocation. Interned keys +// live for the process, evicted nodes' keys included; the table is bounded +// (internMax), and past it a pubkey is returned uninterned. func resolvedPubkeysFromJSON(s string) []string { if s == "" { return nil @@ -238,35 +242,6 @@ func resolvedPubkeysFromJSON(s string) []string { return extractResolvedPubkeys(unmarshalResolvedPath(s)) } -// resolvedPubkeyStrings holds one copy of each pubkey read from a -// resolved_path. Relay pubkeys repeat across nearly every observation, so -// a pubkey seen before costs no allocation: the lookup by substring does -// not copy, and only a new pubkey is cloned. Bounded so a feed of ever-new -// values cannot grow it without limit; past that, pubkeys are cloned. -var resolvedPubkeyStrings = &pubkeyInterner{m: make(map[string]string)} - -type pubkeyInterner struct { - mu sync.Mutex - m map[string]string -} - -// get returns the held copy of sub, keeping a clone if it is new. sub may -// be a slice of a larger string; the held copy never pins that string. -func (in *pubkeyInterner) get(sub string) string { - in.mu.Lock() - defer in.mu.Unlock() - if v, ok := in.m[sub]; ok { - return v - } - v := strings.Clone(sub) - if len(in.m) < resolvedPubkeyStringsMax { - in.m[v] = v - } - return v -} - -const resolvedPubkeyStringsMax = 1 << 16 - // scanResolvedPath reads s as a JSON array of escape-free strings and nulls // with JSON whitespace between tokens. ok is false for anything else. func scanResolvedPath(s string) (out []string, ok bool) { @@ -314,7 +289,7 @@ func scanResolvedPath(s string) (out []string, ok bool) { if out == nil { out = make([]string, 0, 8) } - out = append(out, resolvedPubkeyStrings.get(s[start:i])) + out = append(out, internStr(s[start:i])) } i++ case strings.HasPrefix(s[i:], "null"): diff --git a/cmd/server/resolved_index.go b/cmd/server/resolved_index.go index fa0f8f4e4..940733dd5 100644 --- a/cmd/server/resolved_index.go +++ b/cmd/server/resolved_index.go @@ -15,7 +15,6 @@ import ( "log" "sort" "strings" - "unsafe" ) // resolvedPubkeyHash computes a fast 64-bit hash for membership index keying. @@ -518,21 +517,18 @@ func (s *PacketStore) resolvedPubkeysForEvictionBatch(txIDs []int) map[int][]str } var txID int - var rpJSON sql.RawBytes + var rpJSON sql.NullString for rows.Next() { if err := rows.Scan(&txID, &rpJSON); err != nil { continue } - if len(rpJSON) == 0 { + if !rpJSON.Valid || rpJSON.String == "" { continue } // De-duplicate as we go: a packet has a handful of keys, so a // scan of its list is cheaper than a map per packet. pks := result[txID] - // rpJSON is the driver's buffer, valid until the next row: read - // it without copying. Nothing keeps a slice of it (a new pubkey - // is cloned; the decoder fallback makes its own strings). - for _, pk := range resolvedPubkeysFromJSON(unsafe.String(&rpJSON[0], len(rpJSON))) { + for _, pk := range resolvedPubkeysFromJSON(rpJSON.String) { dup := false for _, have := range pks { if have == pk { diff --git a/cmd/server/resolved_pubkeys_bench_test.go b/cmd/server/resolved_pubkeys_bench_test.go new file mode 100644 index 000000000..70adcb8bd --- /dev/null +++ b/cmd/server/resolved_pubkeys_bench_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" + "strings" + "testing" +) + +// One resolved_path per observation: 5 relays and 3 unresolved hops, drawn +// from a pool of 2,000 relays, as on a busy instance. +func BenchmarkResolvedPubkeysFromJSON(b *testing.B) { + const pool = 2000 + paths := make([]string, 256) + for i := range paths { + hops := make([]string, 0, 8) + for h := 0; h < 5; h++ { + hops = append(hops, fmt.Sprintf(`"%064x"`, (i*7+h*131)%pool)) + } + hops = append(hops, "null", "null", "null") + paths[i] = "[" + strings.Join(hops, ",") + "]" + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = resolvedPubkeysFromJSON(paths[i%len(paths)]) + } +} + +// An eviction batch of 1,000 packets with 3 observations each. +func BenchmarkResolvedPubkeysForEvictionBatch(b *testing.B) { + s, ids := evictionResolvedStore(b, 1000) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = s.resolvedPubkeysForEvictionBatch(ids) + } +}