diff --git a/changelog.d/fixed-logread-cursor-long-first-line.md b/changelog.d/fixed-logread-cursor-long-first-line.md new file mode 100644 index 0000000000..e19bb38cb8 --- /dev/null +++ b/changelog.d/fixed-logread-cursor-long-first-line.md @@ -0,0 +1 @@ +- `log_read` (admin MCP) and `GET /api/logs/read`: a log file whose first line is longer than 64 KiB can be paged. Paging identifies a file by a digest of its first line. The digest was taken over the whole line when a page was returned but over at most 64 KiB when the cursor was looked up, so the two never matched: following `next_cursor` into such a file always answered `cursor_expired` ("no longer retained"), and only the newest page of that file could be read. This hit `hive.log` and its rotated history when a file opened with a very long record, and the `proxy`, `litellm` and `hive-output` sources whenever a file opened with a runaway line, since those are stored in 64 KiB pieces that are each longer than 64 KiB once wrapped. The digest now covers at most the first 64 KiB of the line on both sides. diff --git a/src/crux/logread/cursor_long_first_line_test.go b/src/crux/logread/cursor_long_first_line_test.go new file mode 100644 index 0000000000..b88c7aa584 --- /dev/null +++ b/src/crux/logread/cursor_long_first_line_test.go @@ -0,0 +1,120 @@ +package logread + +import ( + "fmt" + "strings" + "testing" + "time" +) + +// A file is identified by a digest of its first line. The digest was taken +// over the whole line when a page was cut, and over at most 64 KiB when the +// cursor was looked up, so a cursor into a file whose first line is longer +// than 64 KiB always came back cursor_expired and only the newest page of +// that file could be read. +func TestCursorIntoAFileWithALongFirstLine(t *testing.T) { + for _, n := range []int{60 * 1024, maxHeadBytes - 1, maxHeadBytes, maxHeadBytes + 1, 70 * 1024, 300 * 1024} { + dir := t.TempDir() + lines := []string{rec(0, "INFO", strings.Repeat("A", n))} + lines = append(lines, recs(1, 300)...) + writePlain(t, dir, HiveLogFilename, lines) + opts := Options{LogDir: dir} + page1 := Read(Request{Lines: 100}, opts) + if page1.LineCount != 100 || !page1.HasMore || page1.NextCursor == "" { + t.Fatalf("first line of %d bytes: page 1 has %d lines, has_more=%v, cursor=%q", n, page1.LineCount, page1.HasMore, page1.NextCursor) + } + page2 := Read(Request{Lines: 100, Cursor: page1.NextCursor}, opts) + if page2.Status != StatusOK || page2.LineCount != 100 { + t.Errorf("first line of %d bytes: following next_cursor gave status %s with %d lines (%s), want ok with 100", n, page2.Status, page2.LineCount, page2.Reason) + } + } +} + +func TestCursorIntoACompressedFileWithALongFirstLine(t *testing.T) { + dir := t.TempDir() + lines := []string{rec(0, "INFO", strings.Repeat("A", 70*1024))} + lines = append(lines, recs(1, 300)...) + writeGz(t, dir, backupName(0)+".gz", lines) + writePlain(t, dir, HiveLogFilename, recs(300, 310)) + opts := Options{LogDir: dir} + page1 := Read(Request{Lines: 100}, opts) + page2 := Read(Request{Lines: 100, Cursor: page1.NextCursor}, opts) + if page2.Status != StatusOK || page2.LineCount != 100 { + t.Fatalf("following next_cursor into the compressed file gave status %s with %d lines (%s), want ok with 100", page2.Status, page2.LineCount, page2.Reason) + } +} + +// The output sources cut a line that never ends every 64 KiB of text, so the +// record on disk, wrapped in JSON, is longer than 64 KiB. When such a record +// opens the file, the same cursor failure hit proxy, litellm and hive-output. +func TestCursorIntoAnOutputSourceThatOpensWithARunawayLine(t *testing.T) { + dir := t.TempDir() + clock := &fakeClock{t: base} + log := openTestOutput(t, dir, SourceLiteLLM, clock) + w := log.Stream(StreamStdout) + _, _ = w.Write([]byte(strings.Repeat("B", 200*1024) + "\n")) + for i := 0; i < 300; i++ { + clock.advance(time.Second) + fmt.Fprintf(w, "ordinary line %04d\n", i) + } + if err := log.Close(); err != nil { + t.Fatal(err) + } + opts := Options{OutputDir: dir, LocalLiteLLM: true} + page1 := Read(Request{Source: SourceLiteLLM, Lines: 100}, opts) + page2 := Read(Request{Source: SourceLiteLLM, Lines: 100, Cursor: page1.NextCursor}, opts) + if page2.Status != StatusOK || page2.LineCount != 100 { + t.Fatalf("following next_cursor gave status %s with %d lines (%s), want ok with 100", page2.Status, page2.LineCount, page2.Reason) + } +} + +// Every page of such a file is reachable, and no line is lost or repeated. +func TestPagingThroughAFileWithALongFirstLineReturnsEveryLineOnce(t *testing.T) { + dir := t.TempDir() + lines := []string{rec(0, "INFO", strings.Repeat("A", 70*1024))} + lines = append(lines, recs(1, 250)...) + writePlain(t, dir, HiveLogFilename, lines) + opts := Options{LogDir: dir} + seen := 0 + cursor := "" + for page := 0; page < 10; page++ { + res := Read(Request{Lines: 100, Cursor: cursor}, opts) + if res.Status != StatusOK { + t.Fatalf("page %d: status %s (%s)", page, res.Status, res.Reason) + } + seen += res.LineCount + if !res.HasMore { + break + } + cursor = res.NextCursor + } + if seen != len(lines) { + t.Fatalf("paged through %d lines, the file has %d", seen, len(lines)) + } +} + +func TestHeadOf(t *testing.T) { + long := strings.Repeat("x", maxHeadBytes+10) + for name, tc := range map[string]struct { + data string + want int // length of the head, -1 for none + }{ + "ordinary line": {"abc\ndef\n", 3}, + "empty first line": {"\nabc\n", 0}, + "no newline yet": {"abc", -1}, + "empty": {"", -1}, + "line of exactly the cap": {long[:maxHeadBytes] + "\nrest", maxHeadBytes}, + "line longer than the cap": {long + "\nrest", maxHeadBytes}, + "cap-sized read, no newline": {long[:maxHeadBytes], maxHeadBytes}, + "short read, no newline": {long[:maxHeadBytes-1], -1}, + "longer read, no newline yet": {long, maxHeadBytes}, + } { + got := headOf([]byte(tc.data)) + switch { + case tc.want < 0 && got != nil: + t.Errorf("%s: head has %d bytes, want none", name, len(got)) + case tc.want >= 0 && (got == nil || len(got) != tc.want): + t.Errorf("%s: head = %d bytes (nil=%v), want %d", name, len(got), got == nil, tc.want) + } + } +} diff --git a/src/crux/logread/files.go b/src/crux/logread/files.go index 35e9d1f65f..e4fd4a82a7 100644 --- a/src/crux/logread/files.go +++ b/src/crux/logread/files.go @@ -157,11 +157,24 @@ func firstLine(fh *os.File, data []byte, base int64) []byte { const maxHeadBytes = 64 * 1024 +// headOf returns the part of a file's first line that identifies the file: +// the line itself, or its first maxHeadBytes when it is longer. The cap must +// be the same whichever way the head is read. A page is cut from a loaded +// region that can hold the whole first line, while a cursor is looked up by +// reading only maxHeadBytes from the start of each file. When the two +// disagreed, a first line longer than maxHeadBytes gave a digest no lookup +// could ever reproduce, and every cursor into that file came back expired. +// Data with no newline that is shorter than maxHeadBytes is a line still +// being written and has no identity yet. func headOf(data []byte) []byte { - if i := indexNewline(data); i >= 0 { - return data[:i] + i := indexNewline(data) + if i < 0 && len(data) < maxHeadBytes { + return nil + } + if i < 0 || i > maxHeadBytes { + i = maxHeadBytes } - return nil + return data[:i] } func indexNewline(b []byte) int { diff --git a/src/docs/design/admin-mcp.md b/src/docs/design/admin-mcp.md index 220ec4b671..740e2fb7b2 100644 --- a/src/docs/design/admin-mcp.md +++ b/src/docs/design/admin-mcp.md @@ -440,7 +440,8 @@ setting. - **Paging.** `next_cursor` pages backwards. It names a history file by its first line's time and digest plus a byte offset, so a rotation (rename, then compression) between two calls neither repeats nor skips a line; a cursor whose file has left retention returns - `status: cursor_expired`. + `status: cursor_expired`. The digest covers at most the first 64 KiB of that line, so a file + that opens with a longer record can be paged like any other. - **Scrubbing.** Each line is scrubbed at read time before filtering: private-key blocks (scrubbed with overlap across the read region, so a key split by a page or rotation boundary, or one with no END marker, is redacted to the end of the result), then one precompiled