From baaa1ed33811963e2cdd42a7214866077a72021a Mon Sep 17 00:00:00 2001 From: onkar717 Date: Thu, 29 Oct 2026 11:03:43 +0530 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix:=20stop=20dropping=20tail=5F?= =?UTF-8?q?500=20from=20pane-timeout=20self-report=20diagnostics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The slogfeed WARN/ERROR tee that feeds fleet self-reports bounds the attribute excerpt it attaches to an observation at 512 bytes, writing attrs in record order and stopping at the first one that would cross the cap. agent.manager's "pane content at timeout" WARN logs head_500 before tail_500, each up to 500 runes (worse with multi-byte box-drawing characters like ❯ and ╰─); head_500 alone routinely fills 512 bytes, so tail_500 — the pane's CURRENT state, and the only field useful for diagnosing a stuck CLI — was dropped every single time. Both hivecommons/hive#14586 and #15311 self-reported this exact WARN with no usable excerpt attached, and #14586 was closed uninvestigated for lack of one. Raise the cap to 4096 bytes, comfortably fitting both excerpts plus the preceding flags, well under the tracker's own 8KiB storage bound. Closes #15311 Hive-Run: hivecommons/hive#15311 Hive-Plan: fix pane-timeout selfreport diagnostics truncation Hive-Spec: selfreport-diagnostics#agent_pane_timeout Signed-off-by: onkar717 --- .../fixed-15311-pane-timeout-diagnostics.md | 1 + src/pkg/fleetreport/slogfeed/slogfeed.go | 15 ++++++- src/pkg/fleetreport/slogfeed/slogfeed_test.go | 42 +++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-15311-pane-timeout-diagnostics.md diff --git a/changelog.d/fixed-15311-pane-timeout-diagnostics.md b/changelog.d/fixed-15311-pane-timeout-diagnostics.md new file mode 100644 index 0000000000..206cc7c263 --- /dev/null +++ b/changelog.d/fixed-15311-pane-timeout-diagnostics.md @@ -0,0 +1 @@ +- The fleet self-report feed's diagnostics excerpt is now bounded at 4096 bytes instead of 512: the `agent-runtime` "pane content at timeout" warning logs a `head_500` attribute before `tail_500`, and at the old cap `head_500` alone (worse with multi-byte box-drawing characters) filled the budget and silently dropped `tail_500` — the pane's current state and the only field useful for diagnosing a stuck CLI — every time the warning fired. Self-reports for `agent_pane_timeout` now carry a usable excerpt ([#15311](https://github.com/hivecommons/hive/issues/15311), [#14586](https://github.com/hivecommons/hive/issues/14586)). diff --git a/src/pkg/fleetreport/slogfeed/slogfeed.go b/src/pkg/fleetreport/slogfeed/slogfeed.go index 6f2152a444..33163bd0f7 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed.go @@ -26,7 +26,20 @@ const ( maxKeyBytes = 200 // maxDiagnosticsBytes bounds the attribute excerpt attached to the // observation; the tracker scrubs and bounds again on storage. - maxDiagnosticsBytes = 512 + // + // Sized to outlive agent.manager's "pane content at timeout" WARN + // (src/crux/agent/manager.go), whose head_500/tail_500 attrs are each up + // to 500 runes and can run ~1500 bytes apiece when the pane is full of + // multi-byte box-drawing characters (❯, ╰─, ...). diagnostics below + // writes attrs in record order and stops at the first one that would + // cross this cap, so a value too small for head_500 alone drops tail_500 + // — the pane's CURRENT state, and the only field useful for diagnosing a + // stuck CLI — every single time. That is exactly what happened at 512 + // bytes: hivecommons/hive#14586 and #15311 both self-reported this WARN + // with no usable excerpt attached. 4096 comfortably fits both fields plus + // the preceding flags with room to spare, while staying well under the + // tracker's own 8KiB storage bound. + maxDiagnosticsBytes = 4096 // minLevel is the lowest level the feed observes. minLevel = slog.LevelWarn ) diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index a69a48a7cd..937f993b38 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -76,6 +76,48 @@ func TestHandlerWithAttrsCarriesBoundAttrsIntoDiagnostics(t *testing.T) { } } +// TestDiagnosticsKeepsBothPaneTimeoutExcerpts regression-tests +// hivecommons/hive#14586 and #15311: the "pane content at timeout" WARN +// (src/crux/agent/manager.go) logs head_500 before tail_500, each up to 500 +// runes. At the old 512-byte cap, head_500 alone (worse with multi-byte +// box-drawing characters) filled the budget and diagnostics() dropped +// tail_500 — the pane's current state and the only field useful for +// diagnosing a stuck CLI — before it was ever written. +func TestDiagnosticsKeepsBothPaneTimeoutExcerpts(t *testing.T) { + logger, seen, _ := newFeed(t) + head := strings.Repeat("❯", 500) // worst case: 500 runes, 3 bytes each + tail := strings.Repeat("z", 480) + "TAIL-MARKER-PRESENT" + logger.Warn("pane content at timeout", + "agent", "quality", + "len", 28517, + "has_goose_ready", false, + "has_enter", false, + "has_arrow", true, + "has_question_form", false, + "has_bob_placeholder", false, + "has_codex_ready", false, + "head_500", head, + "tail_500", tail, + ) + if len(*seen) != 1 { + t.Fatalf("observed = %+v, want exactly one", *seen) + } + got := (*seen)[0] + if got.class != fleetreport.ClassAgentPaneTimeout { + t.Fatalf("class = %q, want %q", got.class, fleetreport.ClassAgentPaneTimeout) + } + d := got.diagnostics + if !strings.Contains(d, "has_codex_ready=false") { + t.Fatalf("diagnostics dropped a flag before head_500: %q", d) + } + if !strings.Contains(d, "head_500=") { + t.Fatalf("diagnostics missing head_500 entirely: %q", d) + } + if !strings.Contains(d, "TAIL-MARKER-PRESENT") { + t.Fatalf("diagnostics dropped tail_500 (the pane's current state): %q", d) + } +} + func TestKeyBoundsAndNormalises(t *testing.T) { a := Key("request failed for owner/repo-1 after 3 attempts (id 0123456789abcdef)") b := Key("request failed for owner/other-9 after 17 attempts (id fedcba9876543210)")