Skip to content

Commit 72832dd

Browse files
aksOpsclaude
andauthored
ci: gate scale tests behind scale build tag + nightly workflow (#75)
Before: three long-running regression tests used `testing.Short()` guards to stay out of `go test -short` local runs, but CI never passed `-short` so they ran on every PR: - TestImportTar_TotalBytesCap (~8s, 520 MB tar payload) - TestScale_1000Notes (~10s, 1k file writes) - TestHNSW_Recall10k (~55s, 10k-vector index build) After: each lives in a dedicated `_scale_test.go` file behind `//go:build scale` and is compiled only when the tag is passed. PR CI drops ~70s per run; correctness coverage stays intact via the new `nightly.yml` workflow that runs `-tags "sqlite_fts5 scale"` at 06:00 UTC and on workflow_dispatch. Side effects: - The old `if raceEnabled { t.Skip }` guard in TestHNSW_Recall10k is retired because nightly runs the scale tests without `-race` (workload is sequential — concurrency coverage lives in TestHNSW_ConcurrentAddSearch, which still runs under -race on every PR). - normalizeVec moved into hnsw_recall_scale_test.go (its only caller) so default test builds no longer carry dead helpers. Closes #62, #63, #64. Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 3169ccd commit 72832dd

7 files changed

Lines changed: 220 additions & 146 deletions

File tree

.github/workflows/nightly.yml

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: nightly
2+
3+
# Runs scale-tagged tests (large payloads, 10k HNSW recall benchmark,
4+
# 1000-note scale) that are excluded from every PR's CI to keep the
5+
# feedback loop fast. Also usable via workflow_dispatch for one-off
6+
# checks after a perf-sensitive change.
7+
on:
8+
schedule:
9+
- cron: "0 6 * * *" # 06:00 UTC daily
10+
workflow_dispatch:
11+
12+
permissions: read-all
13+
14+
jobs:
15+
scale-tests:
16+
name: go scale tests
17+
runs-on: ubuntu-latest
18+
permissions:
19+
contents: read
20+
env:
21+
CGO_ENABLED: "1"
22+
steps:
23+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
24+
25+
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
26+
with:
27+
go-version-file: go.mod
28+
29+
- name: Go build cache
30+
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
31+
with:
32+
path: |
33+
~/.cache/go-build
34+
~/go/pkg/mod
35+
key: ${{ runner.os }}-go-scale-${{ hashFiles('**/go.sum') }}
36+
restore-keys: |
37+
${{ runner.os }}-go-scale-
38+
39+
# Scale tests need the embedded UI; build a minimal ui/dist so
40+
# //go:embed is satisfied. We don't care about the frontend bundle
41+
# contents here — just that something compiles.
42+
- name: Seed ui/dist placeholder
43+
run: |
44+
mkdir -p ui/dist
45+
printf '<!doctype html><title>nightly</title>' > ui/dist/index.html
46+
47+
- name: go test scale (no -race; workloads are sequential)
48+
run: |
49+
CGO_ENABLED=1 go test \
50+
-tags "sqlite_fts5 scale" \
51+
-timeout 1200s \
52+
$(go list ./... | grep -v /ui/node_modules/)

internal/api/notes_import_limits_test.go

Lines changed: 0 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,8 @@ import (
55
"bytes"
66
"compress/gzip"
77
"fmt"
8-
"io"
98
"net/http"
109
"net/http/httptest"
11-
"strings"
1210
"testing"
1311
)
1412

@@ -73,49 +71,3 @@ func TestImportTar_EntryCountCap(t *testing.T) {
7371
}
7472
}
7573

76-
// TestImportTar_TotalBytesCap is a regression test for P0-3.
77-
// A tar whose total uncompressed bytes across entries exceed
78-
// MaxImportTotalBytes must be rejected with 413.
79-
func TestImportTar_TotalBytesCap(t *testing.T) {
80-
if testing.Short() {
81-
// TODO(#62): large-tar import test skipped under -short; tracked in flake-register.
82-
t.Skip("skipping large-tar test in -short mode")
83-
}
84-
h, slug, _ := setupNotesRouter(t)
85-
86-
// Each entry is just under MaxNoteBytes (10 MB). Two 256 MB entries
87-
// would still fit under MaxImportTotalBytes (500 MB); we need > 500
88-
// MB total. Use 52 entries × 10 MB = 520 MB — exceeds the cap.
89-
// Build each entry's body once and reuse.
90-
perEntry := 10 * 1024 * 1024 // 10 MB, equal to MaxNoteBytes
91-
// Use slightly less to satisfy per-entry cap but still accumulate
92-
// fast.
93-
body := make([]byte, perEntry-1)
94-
for i := range body {
95-
body[i] = 'x'
96-
}
97-
entriesNeeded := int(MaxImportTotalBytes/int64(perEntry-1)) + 3
98-
entries := make([]tarEntry, entriesNeeded)
99-
for i := range entriesNeeded {
100-
entries[i] = tarEntry{
101-
name: fmt.Sprintf("big-%03d.md", i),
102-
body: body,
103-
}
104-
}
105-
tarBytes := newTarGz(t, entries)
106-
req := httptest.NewRequest(http.MethodPost,
107-
"/api/projects/"+slug+"/import", bytes.NewReader(tarBytes))
108-
req.Header.Set("Content-Type", "application/gzip")
109-
rec := httptest.NewRecorder()
110-
h.ServeHTTP(rec, req)
111-
112-
if rec.Code != http.StatusRequestEntityTooLarge {
113-
t.Fatalf("expected 413 for over-total-bytes tar, got %d body=%s",
114-
rec.Code, rec.Body.String())
115-
}
116-
if !strings.Contains(strings.ToLower(rec.Body.String()), "total") &&
117-
!strings.Contains(strings.ToLower(rec.Body.String()), "bytes") {
118-
t.Logf("body=%s", rec.Body.String())
119-
}
120-
_ = io.EOF
121-
}
Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
//go:build scale
2+
3+
package api
4+
5+
import (
6+
"bytes"
7+
"fmt"
8+
"io"
9+
"net/http"
10+
"net/http/httptest"
11+
"strings"
12+
"testing"
13+
)
14+
15+
// TestImportTar_TotalBytesCap is a regression test for P0-3 kept behind
16+
// the `scale` build tag. It allocates ~520 MB of tar payload; the
17+
// dedicated nightly workflow runs it via `-tags "sqlite_fts5 scale"`.
18+
// Default PR CI does not compile this file.
19+
func TestImportTar_TotalBytesCap(t *testing.T) {
20+
h, slug, _ := setupNotesRouter(t)
21+
22+
// Each entry is just under MaxNoteBytes (10 MB). Two 256 MB entries
23+
// would still fit under MaxImportTotalBytes (500 MB); we need > 500
24+
// MB total. Use 52 entries × 10 MB = 520 MB — exceeds the cap.
25+
// Build each entry's body once and reuse.
26+
perEntry := 10 * 1024 * 1024 // 10 MB, equal to MaxNoteBytes
27+
// Use slightly less to satisfy per-entry cap but still accumulate
28+
// fast.
29+
body := make([]byte, perEntry-1)
30+
for i := range body {
31+
body[i] = 'x'
32+
}
33+
entriesNeeded := int(MaxImportTotalBytes/int64(perEntry-1)) + 3
34+
entries := make([]tarEntry, entriesNeeded)
35+
for i := range entriesNeeded {
36+
entries[i] = tarEntry{
37+
name: fmt.Sprintf("big-%03d.md", i),
38+
body: body,
39+
}
40+
}
41+
tarBytes := newTarGz(t, entries)
42+
req := httptest.NewRequest(http.MethodPost,
43+
"/api/projects/"+slug+"/import", bytes.NewReader(tarBytes))
44+
req.Header.Set("Content-Type", "application/gzip")
45+
rec := httptest.NewRecorder()
46+
h.ServeHTTP(rec, req)
47+
48+
if rec.Code != http.StatusRequestEntityTooLarge {
49+
t.Fatalf("expected 413 for over-total-bytes tar, got %d body=%s",
50+
rec.Code, rec.Body.String())
51+
}
52+
if !strings.Contains(strings.ToLower(rec.Body.String()), "total") &&
53+
!strings.Contains(strings.ToLower(rec.Body.String()), "bytes") {
54+
t.Logf("body=%s", rec.Body.String())
55+
}
56+
_ = io.EOF
57+
}

internal/notes/notes_scale_test.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
//go:build scale
2+
3+
package notes
4+
5+
import (
6+
"fmt"
7+
"testing"
8+
)
9+
10+
// TestScale_1000Notes writes 1000 notes across 10 buckets and verifies
11+
// the key listing. Gated behind the `scale` build tag so default PR CI
12+
// stays fast; the nightly workflow runs it via `-tags "sqlite_fts5 scale"`.
13+
func TestScale_1000Notes(t *testing.T) {
14+
dir := t.TempDir()
15+
for i := 0; i < 1000; i++ {
16+
k := fmt.Sprintf("bucket%d/note%d", i%10, i)
17+
if err := Write(dir, &Note{Key: k, Content: "x"}); err != nil {
18+
t.Fatalf("write %d: %v", i, err)
19+
}
20+
}
21+
keys, err := ListKeys(dir)
22+
if err != nil {
23+
t.Fatal(err)
24+
}
25+
if len(keys) != 1000 {
26+
t.Errorf("expected 1000, got %d", len(keys))
27+
}
28+
}

internal/notes/notes_test.go

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -225,27 +225,6 @@ func TestUnicodeKey(t *testing.T) {
225225
}
226226
}
227227

228-
func TestScale_1000Notes(t *testing.T) {
229-
if testing.Short() {
230-
// TODO(#63): 1000-note scale test skipped under -short; tracked in flake-register.
231-
t.Skip("skipping 1000-note scale test in -short mode")
232-
}
233-
dir := t.TempDir()
234-
for i := 0; i < 1000; i++ {
235-
k := fmt.Sprintf("bucket%d/note%d", i%10, i)
236-
if err := Write(dir, &Note{Key: k, Content: "x"}); err != nil {
237-
t.Fatalf("write %d: %v", i, err)
238-
}
239-
}
240-
keys, err := ListKeys(dir)
241-
if err != nil {
242-
t.Fatal(err)
243-
}
244-
if len(keys) != 1000 {
245-
t.Errorf("expected 1000, got %d", len(keys))
246-
}
247-
}
248-
249228
func TestFrontmatterPreserved(t *testing.T) {
250229
dir := t.TempDir()
251230
n := &Note{
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
//go:build scale
2+
3+
package vectorindex
4+
5+
import (
6+
"fmt"
7+
"math/rand"
8+
"testing"
9+
)
10+
11+
// normalizeVec returns v scaled to unit L2 norm (new slice). Lives in
12+
// this scale-tagged file because TestHNSW_Recall10k is its only caller.
13+
func normalizeVec(v []float32) []float32 {
14+
var s float32
15+
for _, x := range v {
16+
s += x * x
17+
}
18+
if s == 0 {
19+
return v
20+
}
21+
n := sqrt32(s)
22+
out := make([]float32, len(v))
23+
for i := range v {
24+
out[i] = v[i] / n
25+
}
26+
return out
27+
}
28+
29+
// TestHNSW_Recall10k builds a 10k-vector index and verifies recall@10
30+
// stays above 0.95 across 20 query probes. The workload is fully
31+
// sequential — the race detector has nothing to catch here —
32+
// so nightly invokes it WITHOUT -race. Concurrency correctness is
33+
// covered by TestHNSW_ConcurrentAddSearch, which runs on every PR.
34+
//
35+
// Gated behind the `scale` build tag; the nightly workflow runs it via
36+
// `-tags "sqlite_fts5 scale"`.
37+
func TestHNSW_Recall10k(t *testing.T) {
38+
const (
39+
n = 10_000
40+
dim = 384
41+
q = 20 // number of query probes
42+
k = 10
43+
)
44+
rng := rand.New(rand.NewSource(7))
45+
// Higher construction/search ef for a strong recall benchmark; the
46+
// default (16/200/50) hits ~0.85 on random vectors which is noisy.
47+
idx := NewHNSW(32, 400, 400)
48+
vecs := make(map[string][]float32, n)
49+
for i := 0; i < n; i++ {
50+
id := fmt.Sprintf("v%d", i)
51+
v := normalizeVec(randomVec(rng, dim))
52+
vecs[id] = v
53+
if err := idx.Add(id, v); err != nil {
54+
t.Fatal(err)
55+
}
56+
}
57+
58+
var totalRecall float64
59+
for qi := 0; qi < q; qi++ {
60+
qv := normalizeVec(randomVec(rng, dim))
61+
gold := bruteForceTopK(qv, vecs, k)
62+
hits, err := idx.Search(qv, k)
63+
if err != nil {
64+
t.Fatal(err)
65+
}
66+
goldSet := map[string]bool{}
67+
for _, id := range gold {
68+
goldSet[id] = true
69+
}
70+
matches := 0
71+
for _, h := range hits {
72+
if goldSet[h.ID] {
73+
matches++
74+
}
75+
}
76+
totalRecall += float64(matches) / float64(k)
77+
}
78+
recall := totalRecall / float64(q)
79+
t.Logf("HNSW recall@10 over %d queries (N=%d, dim=%d) = %.3f", q, n, dim, recall)
80+
if recall < 0.95 {
81+
t.Fatalf("recall@10 = %.3f, want >= 0.95", recall)
82+
}
83+
}

0 commit comments

Comments
 (0)