Skip to content

Commit e311d99

Browse files
committed
perf(graph): cap Kuzu BufferPoolSize and MaxNumThreads by default (Task A3)
kuzu.DefaultSystemConfig() allocates 80% of system RAM as the buffer pool (~12 GiB on a 15 GiB host) before any enrich work runs. Combined with Go-side enricher memory that's enough to OOM the process. The default also allocates full GOMAXPROCS worth of internal threads, amplifying COPY-side working set. Adds OpenOptions struct + OpenWithOptions(path, opts). Open(path) now applies safe defaults via OpenWithOptions(path, OpenOptions{}): - BufferPoolBytes: 2 GiB (DefaultBufferPoolBytes) - MaxThreads: min(4, GOMAXPROCS) OpenReadOnly is unchanged externally (same signature) but routes through OpenWithOptions internally — read paths inherit the same buffer pool cap (2 GiB is plenty for read-side caching at our graph scale). Plan: docs/superpowers/plans/2026-05-13-enrich-oom-fix.md Task A3. Future polish: surface --max-buffer-pool and --copy-threads CLI flags for power-user tuning (deferred). Verification: - go test ./internal/graph/... -count=1: 44 pass - go test ./... -count=1: 876 pass
1 parent 21f07d8 commit e311d99

1 file changed

Lines changed: 71 additions & 19 deletions

File tree

go/internal/graph/store.go

Lines changed: 71 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,60 @@ import (
1515
"fmt"
1616
"os"
1717
"path/filepath"
18+
"runtime"
1819
"sync"
1920
"time"
2021

2122
kuzu "github.com/kuzudb/go-kuzu"
2223
)
2324

25+
// DefaultBufferPoolBytes caps Kuzu's buffer pool to 2 GiB by default.
26+
// kuzu.DefaultSystemConfig() allocates 80% of system RAM (~12 GiB on a 15
27+
// GiB host) before any Go-side enrich work runs, leaving insufficient
28+
// headroom for the in-memory enricher pipeline. 2 GiB is enough for
29+
// real-world graphs at ~/projects/-scale (~430k nodes / ~300k edges) while
30+
// keeping the host OOM bar well below ceiling.
31+
const DefaultBufferPoolBytes uint64 = 2 << 30
32+
33+
// defaultMaxThreads returns the per-query thread cap for Kuzu — bounded so
34+
// COPY FROM's working set scales with parallelism in a controlled way.
35+
// min(4, GOMAXPROCS): keeps headroom even on small hosts; 4 is enough to
36+
// saturate IO+CPU for our COPY shape.
37+
func defaultMaxThreads() uint64 {
38+
n := runtime.GOMAXPROCS(0)
39+
if n > 4 {
40+
n = 4
41+
}
42+
if n < 1 {
43+
n = 1
44+
}
45+
return uint64(n)
46+
}
47+
48+
// OpenOptions tunes how Open and OpenReadOnly wire the underlying Kuzu
49+
// SystemConfig. Zero-valued fields fall back to safe defaults documented
50+
// alongside each field.
51+
type OpenOptions struct {
52+
// BufferPoolBytes caps Kuzu's buffer pool in bytes. Zero -> DefaultBufferPoolBytes.
53+
BufferPoolBytes uint64
54+
// MaxThreads caps Kuzu's per-query parallelism. Zero -> defaultMaxThreads().
55+
MaxThreads uint64
56+
// ReadOnly opens the database in read-only mode.
57+
ReadOnly bool
58+
// QueryTimeout, if > 0, sets the per-query wall-clock timeout.
59+
QueryTimeout time.Duration
60+
}
61+
62+
func (o OpenOptions) resolved() OpenOptions {
63+
if o.BufferPoolBytes == 0 {
64+
o.BufferPoolBytes = DefaultBufferPoolBytes
65+
}
66+
if o.MaxThreads == 0 {
67+
o.MaxThreads = defaultMaxThreads()
68+
}
69+
return o
70+
}
71+
2472
// Store is the embedded Kuzu graph store facade. It owns one Kuzu database
2573
// and a single long-lived connection. The zero value is not usable — call
2674
// Open or OpenReadOnly to construct.
@@ -32,14 +80,26 @@ type Store struct {
3280
readOnly bool
3381
}
3482

35-
// Open creates or opens a Kuzu database at the given directory path. Kuzu
36-
// itself creates the directory if it does not exist; we ensure the parent
37-
// exists so a fresh `.codeiq/graph/codeiq.kuzu/` works on first run.
83+
// Open creates or opens a Kuzu database with safe default OpenOptions
84+
// (capped BufferPoolBytes + MaxThreads). For tuning, see OpenWithOptions.
3885
func Open(path string) (*Store, error) {
86+
return OpenWithOptions(path, OpenOptions{})
87+
}
88+
89+
// OpenWithOptions creates or opens a Kuzu database, applying any non-zero
90+
// fields of opts. Zero-valued fields fall back to safe defaults — see
91+
// OpenOptions and DefaultBufferPoolBytes.
92+
func OpenWithOptions(path string, opts OpenOptions) (*Store, error) {
3993
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
4094
return nil, fmt.Errorf("graph: mkdir parent: %w", err)
4195
}
96+
opts = opts.resolved()
4297
sys := kuzu.DefaultSystemConfig()
98+
sys.BufferPoolSize = opts.BufferPoolBytes
99+
sys.MaxNumThreads = opts.MaxThreads
100+
if opts.ReadOnly {
101+
sys.ReadOnly = true
102+
}
43103
db, err := kuzu.OpenDatabase(path, sys)
44104
if err != nil {
45105
return nil, fmt.Errorf("graph: open db: %w", err)
@@ -49,7 +109,10 @@ func Open(path string) (*Store, error) {
49109
db.Close()
50110
return nil, fmt.Errorf("graph: open conn: %w", err)
51111
}
52-
return &Store{db: db, conn: conn, path: path}, nil
112+
if opts.QueryTimeout > 0 {
113+
conn.SetTimeout(uint64(opts.QueryTimeout / time.Millisecond))
114+
}
115+
return &Store{db: db, conn: conn, path: path, readOnly: opts.ReadOnly}, nil
53116
}
54117

55118
// OpenReadOnly opens an existing Kuzu store in read-only mode and sets a
@@ -65,21 +128,10 @@ func Open(path string) (*Store, error) {
65128
// queryTimeout <= 0 disables the per-query timeout. Kuzu interprets the
66129
// timeout in milliseconds; we accept a Go duration for ergonomics.
67130
func OpenReadOnly(path string, queryTimeout time.Duration) (*Store, error) {
68-
sys := kuzu.DefaultSystemConfig()
69-
sys.ReadOnly = true
70-
db, err := kuzu.OpenDatabase(path, sys)
71-
if err != nil {
72-
return nil, fmt.Errorf("graph: open read-only %q: %w", path, err)
73-
}
74-
conn, err := kuzu.OpenConnection(db)
75-
if err != nil {
76-
db.Close()
77-
return nil, fmt.Errorf("graph: open ro conn: %w", err)
78-
}
79-
if queryTimeout > 0 {
80-
conn.SetTimeout(uint64(queryTimeout / time.Millisecond))
81-
}
82-
return &Store{db: db, conn: conn, path: path, readOnly: true}, nil
131+
return OpenWithOptions(path, OpenOptions{
132+
ReadOnly: true,
133+
QueryTimeout: queryTimeout,
134+
})
83135
}
84136

85137
// IsReadOnly reports whether the store rejects mutating Cypher.

0 commit comments

Comments
 (0)