Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-logread-cursor-long-first-line.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- `log_read` (admin MCP) and `GET /api/logs/read`: a log file whose first line is longer than 64 KiB can be paged. Paging identifies a file by a digest of its first line. The digest was taken over the whole line when a page was returned but over at most 64 KiB when the cursor was looked up, so the two never matched: following `next_cursor` into such a file always answered `cursor_expired` ("no longer retained"), and only the newest page of that file could be read. This hit `hive.log` and its rotated history when a file opened with a very long record, and the `proxy`, `litellm` and `hive-output` sources whenever a file opened with a runaway line, since those are stored in 64 KiB pieces that are each longer than 64 KiB once wrapped. The digest now covers at most the first 64 KiB of the line on both sides.
120 changes: 120 additions & 0 deletions src/crux/logread/cursor_long_first_line_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
package logread

import (
"fmt"
"strings"
"testing"
"time"
)

// A file is identified by a digest of its first line. The digest was taken
// over the whole line when a page was cut, and over at most 64 KiB when the
// cursor was looked up, so a cursor into a file whose first line is longer
// than 64 KiB always came back cursor_expired and only the newest page of
// that file could be read.
func TestCursorIntoAFileWithALongFirstLine(t *testing.T) {
for _, n := range []int{60 * 1024, maxHeadBytes - 1, maxHeadBytes, maxHeadBytes + 1, 70 * 1024, 300 * 1024} {
dir := t.TempDir()
lines := []string{rec(0, "INFO", strings.Repeat("A", n))}
lines = append(lines, recs(1, 300)...)
writePlain(t, dir, HiveLogFilename, lines)
opts := Options{LogDir: dir}
page1 := Read(Request{Lines: 100}, opts)
if page1.LineCount != 100 || !page1.HasMore || page1.NextCursor == "" {
t.Fatalf("first line of %d bytes: page 1 has %d lines, has_more=%v, cursor=%q", n, page1.LineCount, page1.HasMore, page1.NextCursor)
}
page2 := Read(Request{Lines: 100, Cursor: page1.NextCursor}, opts)
if page2.Status != StatusOK || page2.LineCount != 100 {
t.Errorf("first line of %d bytes: following next_cursor gave status %s with %d lines (%s), want ok with 100", n, page2.Status, page2.LineCount, page2.Reason)
}
}
}

func TestCursorIntoACompressedFileWithALongFirstLine(t *testing.T) {
dir := t.TempDir()
lines := []string{rec(0, "INFO", strings.Repeat("A", 70*1024))}
lines = append(lines, recs(1, 300)...)
writeGz(t, dir, backupName(0)+".gz", lines)
writePlain(t, dir, HiveLogFilename, recs(300, 310))
opts := Options{LogDir: dir}
page1 := Read(Request{Lines: 100}, opts)
page2 := Read(Request{Lines: 100, Cursor: page1.NextCursor}, opts)
if page2.Status != StatusOK || page2.LineCount != 100 {
t.Fatalf("following next_cursor into the compressed file gave status %s with %d lines (%s), want ok with 100", page2.Status, page2.LineCount, page2.Reason)
}
}

// The output sources cut a line that never ends every 64 KiB of text, so the
// record on disk, wrapped in JSON, is longer than 64 KiB. When such a record
// opens the file, the same cursor failure hit proxy, litellm and hive-output.
func TestCursorIntoAnOutputSourceThatOpensWithARunawayLine(t *testing.T) {
dir := t.TempDir()
clock := &fakeClock{t: base}
log := openTestOutput(t, dir, SourceLiteLLM, clock)
w := log.Stream(StreamStdout)
_, _ = w.Write([]byte(strings.Repeat("B", 200*1024) + "\n"))
for i := 0; i < 300; i++ {
clock.advance(time.Second)
fmt.Fprintf(w, "ordinary line %04d\n", i)
}
if err := log.Close(); err != nil {
t.Fatal(err)
}
opts := Options{OutputDir: dir, LocalLiteLLM: true}
page1 := Read(Request{Source: SourceLiteLLM, Lines: 100}, opts)
page2 := Read(Request{Source: SourceLiteLLM, Lines: 100, Cursor: page1.NextCursor}, opts)
if page2.Status != StatusOK || page2.LineCount != 100 {
t.Fatalf("following next_cursor gave status %s with %d lines (%s), want ok with 100", page2.Status, page2.LineCount, page2.Reason)
}
}

// Every page of such a file is reachable, and no line is lost or repeated.
func TestPagingThroughAFileWithALongFirstLineReturnsEveryLineOnce(t *testing.T) {
dir := t.TempDir()
lines := []string{rec(0, "INFO", strings.Repeat("A", 70*1024))}
lines = append(lines, recs(1, 250)...)
writePlain(t, dir, HiveLogFilename, lines)
opts := Options{LogDir: dir}
seen := 0
cursor := ""
for page := 0; page < 10; page++ {
res := Read(Request{Lines: 100, Cursor: cursor}, opts)
if res.Status != StatusOK {
t.Fatalf("page %d: status %s (%s)", page, res.Status, res.Reason)
}
seen += res.LineCount
if !res.HasMore {
break
}
cursor = res.NextCursor
}
if seen != len(lines) {
t.Fatalf("paged through %d lines, the file has %d", seen, len(lines))
}
}

func TestHeadOf(t *testing.T) {
long := strings.Repeat("x", maxHeadBytes+10)
for name, tc := range map[string]struct {
data string
want int // length of the head, -1 for none
}{
"ordinary line": {"abc\ndef\n", 3},
"empty first line": {"\nabc\n", 0},
"no newline yet": {"abc", -1},
"empty": {"", -1},
"line of exactly the cap": {long[:maxHeadBytes] + "\nrest", maxHeadBytes},
"line longer than the cap": {long + "\nrest", maxHeadBytes},
"cap-sized read, no newline": {long[:maxHeadBytes], maxHeadBytes},
"short read, no newline": {long[:maxHeadBytes-1], -1},
"longer read, no newline yet": {long, maxHeadBytes},
} {
got := headOf([]byte(tc.data))
switch {
case tc.want < 0 && got != nil:
t.Errorf("%s: head has %d bytes, want none", name, len(got))
case tc.want >= 0 && (got == nil || len(got) != tc.want):
t.Errorf("%s: head = %d bytes (nil=%v), want %d", name, len(got), got == nil, tc.want)
}
}
}
19 changes: 16 additions & 3 deletions src/crux/logread/files.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,11 +157,24 @@ func firstLine(fh *os.File, data []byte, base int64) []byte {

const maxHeadBytes = 64 * 1024

// headOf returns the part of a file's first line that identifies the file:
// the line itself, or its first maxHeadBytes when it is longer. The cap must
// be the same whichever way the head is read. A page is cut from a loaded
// region that can hold the whole first line, while a cursor is looked up by
// reading only maxHeadBytes from the start of each file. When the two
// disagreed, a first line longer than maxHeadBytes gave a digest no lookup
// could ever reproduce, and every cursor into that file came back expired.
// Data with no newline that is shorter than maxHeadBytes is a line still
// being written and has no identity yet.
func headOf(data []byte) []byte {
if i := indexNewline(data); i >= 0 {
return data[:i]
i := indexNewline(data)
if i < 0 && len(data) < maxHeadBytes {
return nil
}
if i < 0 || i > maxHeadBytes {
i = maxHeadBytes
}
return nil
return data[:i]
}

func indexNewline(b []byte) int {
Expand Down
3 changes: 2 additions & 1 deletion src/docs/design/admin-mcp.md
Original file line number Diff line number Diff line change
Expand Up @@ -440,7 +440,8 @@ setting.
- **Paging.** `next_cursor` pages backwards. It names a history file by its first line's time
and digest plus a byte offset, so a rotation (rename, then compression) between two calls
neither repeats nor skips a line; a cursor whose file has left retention returns
`status: cursor_expired`.
`status: cursor_expired`. The digest covers at most the first 64 KiB of that line, so a file
that opens with a longer record can be paged like any other.
- **Scrubbing.** Each line is scrubbed at read time before filtering: private-key blocks
(scrubbed with overlap across the read region, so a key split by a page or rotation boundary,
or one with no END marker, is redacted to the end of the result), then one precompiled
Expand Down
Loading