fix(runner): stream --ids sources and cap selection to the pinned set

CHANGELOG.md

@@ -0,0 +1,5 @@
+# Changelog
+
+## Unreleased
+
+- Fix unbounded memory use when loading benchmark `--ids` from files or HTTP, including whitespace-padded IDs; document selection limits and preserve full-set JSONL support. Thanks @SebTardif (#47).

cmd/clawscan/main.go

@@ -807,7 +807,8 @@ OpenClaw install policy:
 
 Benchmark command flags:
   --split <name>              Benchmark split. Defaults to benchmark for SkillTrustBench and eval_holdout for clawhub-security-signals.
-  --ids <path-or-url>         Run selected benchmark IDs from a text file or JSONL id source. SkillTrustBench only.
+  --ids <path-or-url>         Run selected benchmark IDs from a streamed text or JSONL source (max 5520 IDs). SkillTrustBench only.
+                             Max 256 bytes per trimmed ID and 256 KiB total ID text, not source size; lines must be under 1 MiB.
   --limit <n>                 Maximum benchmark rows to run. 0 means all rows.
   --offset <n>                Benchmark row offset. Defaults to 0.
   --predictions-output <path> Write benchmark predictions JSONL. Defaults next to --output for clawhub-security-signals.

docs/benchmarks.md

@@ -12,7 +12,13 @@ clawscan benchmark SkillTrustBench \
 ```
 
 Use `--ids <path-or-url>` with SkillTrustBench to run a fixed subset from a
-plain text ID list or JSONL rows with an `id` field.
+plain text ID list or JSONL rows with an `id` field. The loader streams the
+source (file or HTTP) and accepts at most 5,520 unique IDs, the size of the
+pinned SkillTrustBench full set. Each extracted ID may contain at most 256
+bytes, and all retained ID text together may contain at most 256 KiB (262,144
+bytes). Whitespace around IDs is trimmed. These limits apply to the extracted
+IDs, not total source size: the full JSONL list may exceed 256 KiB. Individual
+lines must be smaller than the parser's 1 MiB buffer limit.
 
 ## Available benchmarks
 

internal/runner/benchmark.go

@@ -40,6 +40,18 @@ const (
 	huggingFaceRowsEndpoint       = "https://datasets-server.huggingface.co/rows"
 	huggingFaceRowsPageSize       = 100
 	huggingFaceRowsMaxAttempts    = 6
+	// maxSkillTrustBenchIDSelection is the pinned SkillTrustBench full set
+	// (5,520 cases). --ids is SkillTrustBench-only, so a valid selection
+	// cannot contain more unique IDs than that set.
+	maxSkillTrustBenchIDSelection = 5520
+	// maxBenchmarkIDBytes caps one extracted id. Documented SkillTrustBench
+	// ids are case_NNNNN. The scanner still allows 1 MiB records, so this
+	// stops a hostile source from retaining megabyte-sized unique ids.
+	maxBenchmarkIDBytes = 256
+	// maxBenchmarkIDSelectionBytes caps retained id text (not the JSONL
+	// stream). 256 KiB holds the 5,520-id set with headroom; it is not a
+	// file-size limit (the full JSONL is about 1.3 MiB).
+	maxBenchmarkIDSelectionBytes = 256 * 1024
 )
 
 var huggingFaceRowsRetryDelay = 2 * time.Second
@@ -312,11 +324,12 @@ func LoadBenchmarkIDSelection(source string) (BenchmarkIDSelection, error) {
 	if source == "" {
 		return BenchmarkIDSelection{}, errors.New("--ids source is required")
 	}
-	data, err := readBenchmarkIDSource(source)
+	reader, err := openBenchmarkIDSource(source)
 	if err != nil {
 		return BenchmarkIDSelection{}, err
 	}
-	ids, err := parseBenchmarkIDs(source, data)
+	defer reader.Close()
+	ids, err := parseBenchmarkIDs(source, reader)
 	if err != nil {
 		return BenchmarkIDSelection{}, err
 	}
@@ -329,31 +342,32 @@ func LoadBenchmarkIDSelection(source string) (BenchmarkIDSelection, error) {
 	}, nil
 }
 
-func readBenchmarkIDSource(source string) ([]byte, error) {
+func openBenchmarkIDSource(source string) (io.ReadCloser, error) {
 	if parsed, err := url.Parse(source); err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") {
 		client := &http.Client{Timeout: 60 * time.Second}
 		resp, err := client.Get(source)
 		if err != nil {
 			return nil, fmt.Errorf("read --ids source %s: %w", source, err)
 		}
-		defer resp.Body.Close()
 		if resp.StatusCode < 200 || resp.StatusCode > 299 {
+			resp.Body.Close()
 			return nil, fmt.Errorf("read --ids source %s: HTTP %d", source, resp.StatusCode)
 		}
-		return io.ReadAll(resp.Body)
+		return resp.Body, nil
 	}
-	data, err := os.ReadFile(source)
+	file, err := os.Open(source)
 	if err != nil {
 		return nil, fmt.Errorf("read --ids source %s: %w", source, err)
 	}
-	return data, nil
+	return file, nil
 }
 
-func parseBenchmarkIDs(source string, data []byte) ([]string, error) {
-	scanner := bufio.NewScanner(strings.NewReader(string(data)))
+func parseBenchmarkIDs(source string, reader io.Reader) ([]string, error) {
+	scanner := bufio.NewScanner(reader)
 	scanner.Buffer(make([]byte, 1024), 1024*1024)
 	var ids []string
 	seen := map[string]bool{}
+	retained := 0
 	lineNumber := 0
 	for scanner.Scan() {
 		lineNumber++
@@ -368,8 +382,20 @@ func parseBenchmarkIDs(source string, data []byte) ([]string, error) {
 		if seen[id] {
 			return nil, fmt.Errorf("--ids source %s line %d duplicates benchmark id %s", source, lineNumber, id)
 		}
+		if len(id) > maxBenchmarkIDBytes {
+			return nil, fmt.Errorf("--ids source %s line %d exceeds the %d-byte benchmark id limit", source, lineNumber, maxBenchmarkIDBytes)
+		}
+		if retained+len(id) > maxBenchmarkIDSelectionBytes {
+			return nil, fmt.Errorf("--ids source %s exceeds the %d-byte retained-id budget", source, maxBenchmarkIDSelectionBytes)
+		}
+		// Trimmed text IDs must not retain their potentially large source lines.
+		id = strings.Clone(id)
 		seen[id] = true
 		ids = append(ids, id)
+		retained += len(id)
+		if len(ids) > maxSkillTrustBenchIDSelection {
+			return nil, fmt.Errorf("--ids source %s exceeds the %d-id SkillTrustBench selection limit", source, maxSkillTrustBenchIDSelection)
+		}
 	}
 	if err := scanner.Err(); err != nil {
 		return nil, fmt.Errorf("read --ids source %s: %w", source, err)

internal/runner/runner_test.go

@@ -801,6 +801,146 @@ func TestLoadBenchmarkIDSelectionRejectsBadSources(t *testing.T) {
 	}
 }
 
+func TestLoadBenchmarkIDSelectionAcceptsJSONLLargerThan256KiB(t *testing.T) {
+	payload := oversizedBenchmarkIDJSONL(t, maxSkillTrustBenchIDSelection)
+	if len(payload) <= 256*1024 {
+		t.Fatalf("fixture is %d bytes, want more than 256 KiB", len(payload))
+	}
+
+	path := filepath.Join(t.TempDir(), "ids.jsonl")
+	if err := os.WriteFile(path, payload, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	fileSelection, err := LoadBenchmarkIDSelection(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(fileSelection.IDs) != maxSkillTrustBenchIDSelection {
+		t.Fatalf("file ids = %d, want %d", len(fileSelection.IDs), maxSkillTrustBenchIDSelection)
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write(payload)
+	}))
+	defer server.Close()
+
+	httpSelection, err := LoadBenchmarkIDSelection(server.URL + "/ids.jsonl")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(httpSelection.IDs, fileSelection.IDs) || httpSelection.SHA256 != fileSelection.SHA256 {
+		t.Fatal("HTTP and file selections differ")
+	}
+}
+
+func TestLoadBenchmarkIDSelectionReleasesWhitespacePadding(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "padded-ids.txt")
+	file, err := os.Create(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < 64; i++ {
+		if _, err := fmt.Fprintf(file, "%s case_%05d\n", strings.Repeat(" ", 256*1024), i); err != nil {
+			file.Close()
+			t.Fatal(err)
+		}
+	}
+	if err := file.Close(); err != nil {
+		t.Fatal(err)
+	}
+	runtime.GC()
+	var before, after runtime.MemStats
+	runtime.ReadMemStats(&before)
+	selection, err := LoadBenchmarkIDSelection(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	runtime.GC()
+	runtime.ReadMemStats(&after)
+	runtime.KeepAlive(selection)
+	// The IDs occupy hundreds of bytes; retaining their padded source lines
+	// would keep more than 16 MiB live after collection.
+	if retained := int64(after.HeapAlloc) - int64(before.HeapAlloc); retained > 2*1024*1024 {
+		t.Fatalf("retained %d bytes for %d short IDs", retained, len(selection.IDs))
+	}
+}
+
+func TestLoadBenchmarkIDSelectionClosesRejectedHTTPStream(t *testing.T) {
+	closed := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, strings.Repeat("x", maxBenchmarkIDBytes+1))
+		w.(http.Flusher).Flush()
+		select {
+		case <-r.Context().Done():
+			close(closed)
+		case <-time.After(5 * time.Second):
+		}
+	}))
+	defer server.Close()
+	_, err := LoadBenchmarkIDSelection(server.URL)
+	if err == nil || !strings.Contains(err.Error(), "256-byte benchmark id limit") {
+		t.Fatalf("err = %v, want ID length rejection", err)
+	}
+	select {
+	case <-closed:
+	case <-time.After(5 * time.Second):
+		t.Fatal("rejected stream was not closed before reading the entire response")
+	}
+}
+
+func TestLoadBenchmarkIDSelectionRejectsOversizedRetainedIDs(t *testing.T) {
+	huge := strings.Repeat("a", maxBenchmarkIDBytes+1)
+	path := filepath.Join(t.TempDir(), "huge-id.txt")
+	if err := os.WriteFile(path, []byte(huge+"\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	_, err := LoadBenchmarkIDSelection(path)
+	if err == nil || !strings.Contains(err.Error(), "256-byte benchmark id limit") {
+		t.Fatalf("err = %v, want 256-byte benchmark id limit", err)
+	}
+
+	var body strings.Builder
+	// 2000 IDs * 200 bytes is under the 5,520 count cap but over the
+	// retained-id budget (256 KiB).
+	chunk := strings.Repeat("b", 200)
+	for i := 0; i < 2000; i++ {
+		fmt.Fprintf(&body, "%s-%04d\n", chunk, i)
+	}
+	aggPath := filepath.Join(t.TempDir(), "agg-ids.txt")
+	if err := os.WriteFile(aggPath, []byte(body.String()), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	_, err = LoadBenchmarkIDSelection(aggPath)
+	if err == nil || !strings.Contains(err.Error(), "262144-byte retained-id budget") {
+		t.Fatalf("err = %v, want 262144-byte retained-id budget", err)
+	}
+}
+
+func TestLoadBenchmarkIDSelectionRejectsMoreIDsThanPinnedSet(t *testing.T) {
+	var body strings.Builder
+	for i := 0; i < maxSkillTrustBenchIDSelection+1; i++ {
+		fmt.Fprintf(&body, "case_%05d\n", i)
+	}
+	path := filepath.Join(t.TempDir(), "ids.txt")
+	if err := os.WriteFile(path, []byte(body.String()), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	_, err := LoadBenchmarkIDSelection(path)
+	if err == nil || !strings.Contains(err.Error(), "5520-id") {
+		t.Fatalf("err = %v, want 5520-id selection limit", err)
+	}
+}
+
+func oversizedBenchmarkIDJSONL(t *testing.T, count int) []byte {
+	t.Helper()
+	var body strings.Builder
+	pad := strings.Repeat("x", 700)
+	for i := 0; i < count; i++ {
+		fmt.Fprintf(&body, `{"id":"case_%05d","judgment":"normal","pad":"%s"}`+"\n", i, pad)
+	}
+	return []byte(body.String())
+}
+
 func TestRunSkillTrustBenchBenchmarkRejectsMissingSelectedID(t *testing.T) {
 	dir := t.TempDir()
 	idsPath := filepath.Join(dir, "ids.txt")

skills/clawscan-cli/SKILL.md

@@ -213,7 +213,11 @@ SkillTrustBench uses split `benchmark`. The first live run downloads and caches
 into temporary scan targets.
 
 Use `--ids <path-or-url>` with SkillTrustBench to run a fixed subset from a
-plain text file with one ID per line or JSONL rows with an `id` field. `--ids`
+plain text file with one ID per line or JSONL rows with an `id` field. The
+source is streamed and may contain at most 5,520 unique IDs, at most 256 bytes
+per trimmed ID, and at most 256 KiB (262,144 bytes) of retained ID text. These
+are selection limits, not a total source-size limit; JSONL sources may exceed
+256 KiB. Individual lines must be smaller than the parser's 1 MiB buffer limit. `--ids`
 preserves source order, records `idsSource`, `idsCount`, and `idsSha256` in the
 artifact, and is mutually exclusive with `--limit` and `--offset`.