From 39049010a472b99ea5c81dd30b53cae0ffa83453 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 23:02:29 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Meetings:=20don't=20restart-loop?= =?UTF-8?q?=20the=20Spektacular=20stream=20child=20when=20live=20mode=20is?= =?UTF-8?q?n't=20available?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pinned spektacular (0.31.0) has no `meeting` command (hivecommons/spektacular#126 is unreleased), so every live meeting child printed cobra's unknown-command envelope and exited 1. The stream restarted it five times per meeting, each restart a WARN, which the slog feed self-reported as unclassified_error_repeated (#18630), and the session warned once per transcript line it could not forward. - spektacular.Stream gains an optional Probe (`meeting --help`), run once per installed binary; an unknown command marks the stream Unsupported: no child is started or restarted, one INFO line per binary. A child whose own exit output says unknown command is not restarted either. - Restart lines are WARN at most once per BackoffMax window (INFO otherwise) and map to a new log-fed class spek_stream_unavailable. - The live session treats an unsupported Spek as off from the start (panel status unchanged), stops forwarding once Spek is gone, and warns about a failed forward only on the transition to paused. Fixes #18630 Signed-off-by: Andy Anderson --- .../fixed-18630-spek-stream-restart-loop.md | 1 + src/cmd/hive/meetingswire.go | 10 +- src/docs/fleet-report.md | 1 + src/pkg/fleetreport/runtime.go | 6 + src/pkg/fleetreport/runtime_test.go | 3 + src/pkg/meetings/live/session.go | 41 ++++- .../meetings/live/spek_unavailable_test.go | 95 ++++++++++ src/pkg/spektacular/stream.go | 167 +++++++++++++++++- src/pkg/spektacular/stream_test.go | 149 ++++++++++++++++ 9 files changed, 458 insertions(+), 15 deletions(-) create mode 100644 changelog.d/fixed-18630-spek-stream-restart-loop.md create mode 100644 src/pkg/meetings/live/spek_unavailable_test.go diff --git a/changelog.d/fixed-18630-spek-stream-restart-loop.md b/changelog.d/fixed-18630-spek-stream-restart-loop.md new file mode 100644 index 0000000000..e10f6be96d --- /dev/null +++ b/changelog.d/fixed-18630-spek-stream-restart-loop.md @@ -0,0 +1 @@ +- Meetings no longer restart-loop the Spektacular live child when the installed spektacular has no `meeting` command (before hivecommons/spektacular#126): hive probes `spektacular meeting --help` once per installed binary, keeps live mode off without starting the child, stops forwarding transcript lines to it, and the Meetings panel still says notes and clarifying questions are off. A supported child that keeps exiting still restarts with capped exponential backoff, warns at most once per backoff window, and is self-reported as `spek_stream_unavailable` instead of `unclassified_error_repeated` ([#18630](https://github.com/hivecommons/hive/issues/18630)). diff --git a/src/cmd/hive/meetingswire.go b/src/cmd/hive/meetingswire.go index b14aaef683..e634cbd051 100644 --- a/src/cmd/hive/meetingswire.go +++ b/src/cmd/hive/meetingswire.go @@ -160,6 +160,12 @@ func wireMeetingsManager(cfg *config.Config, sup quorumSupervisor, logger *slog. return mgr, store } +// meetingSpekProbeArgs checks once per installed binary that spektacular +// has the live `meeting` command (hivecommons/spektacular#126); a release +// without it answers "unknown command" and live mode stays off without a +// child restart loop (hivecommons/hive#18630). +var meetingSpekProbeArgs = []string{"meeting", "--help"} + // meetingSpekArgs is the contract's live-mode command line // (quorum.contract/v1 section 2). func meetingSpekArgs(m meetings.Meeting, contextDir string) []string { @@ -205,7 +211,9 @@ func wireMeetingsLive(cfg *config.Config, mgr *meetings.Manager, store *meetings Store: store, NewSpek: func(mt meetings.Meeting) live.Spek { env := meetingSpekEnv(os.Getenv, llm, hiveBin, meetingUsageFile(store, mt.ID), slotsDir, maxTokens) - return spektacular.StreamExec(binary, meetingSpekArgs(mt, filepath.Join(store.Dir(), mt.ID)), spektacular.StreamOptions{Logger: logger, Env: env}) + return spektacular.StreamExec(binary, meetingSpekArgs(mt, filepath.Join(store.Dir(), mt.ID)), spektacular.StreamOptions{ + Logger: logger, Env: env, Probe: meetingSpekProbeArgs, + }) }, SpekMode: llm.Label(), UsageFile: func(id string) string { diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 319138771b..3dc1f7a590 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -155,6 +155,7 @@ title is `[selfreport] : `. | `github_fetch_transient` | a non-critical GitHub fetch failed and the feature degrades gracefully (`channel commit date: fetch failed — the row will render without a timestamp`, `pkg/hub/channel_commit_date.go`) — typically a rate limit or transient API error, not a hive defect. Reported only when persistent: log-fed, same ≥5-in-10-min repeat threshold and idle clear as `agent_pane_timeout`. The sibling `SHA poll: commit message fetch non-200` has its own class, `sha_poll_commit_message_unavailable` | low | | `label_provisioning_failed` | provisioning the issue stage labels on a repo failed (`issue stage labels: provisioning failed`, `pkg/github/issue_stage_labels.go`) — typically the App lacking label permission on that repo or a transient API error; diagnostics carry the repo and scrubbed error. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `agent_cli_restart_loop` | the agent CLI crashed (bare shell) or sat on a consent screen and the harness restarted it before the kick (`agent CLI crashed or stuck on consent screen, restarting before kick`, `crux/agent/manager_kick.go`, `kick_async.go`), repeated ≥5 times in 10 min — a restart loop, not transient noise. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `spek_stream_unavailable` | a Spektacular live meeting child (`spektacular meeting`, `pkg/spektacular/stream.go`) keeps exiting and is restarted with exponential backoff (`[spektacular] stream child exited; restarting`), or gave up after its restart cap (`[spektacular] stream child keeps exiting; giving up`). The restart line is a WARN at most once per backoff window and INFO otherwise. A spektacular release without the `meeting` command (before hivecommons/spektacular#126) is not this class: the stream probes `spektacular meeting --help` once per installed binary, logs one INFO and never starts or restarts the child, and the Meetings panel says notes and clarifying questions are off ([#18630](https://github.com/hivecommons/hive/issues/18630)). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `copilot_cli_hung` | a Copilot agent showed none of the CLI's startup chrome (input prompt, help hint, login request) 180 s after launch, outside kick delivery, so the hang detector killed its session, ran bare `copilot` to capture the error and relaunched the agent (`copilot hung with no CLI prompt, running diagnostic`, `crux/agent/manager_poll.go`), or that diagnostic saw no output before its own deadline and relaunched anyway (`diagnostic: timed out waiting for copilot error output`, `crux/agent/manager_credentials.go`; the line carries the pane's `output_tail`). The key is the message, so ≥5 in 10 min counts every Copilot agent on the hive together: its Copilot agents keep failing to reach their prompt. Typical causes are a stale Copilot token the CLI hangs on (the diagnostic restores or clears it when it sees an auth error), a blocked network path to the Copilot API, or a CLI build whose startup screen the detector does not recognise (`cliReadyIndicators`, `crux/agent/pane_classify.go`). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `proxy_connect_dial_failed` | the hive proxy could not dial the upstream host of a CONNECT request (`proxy: CONNECT dial failed`, `crux/proxy/github_proxy_connect.go`) or of a Copilot usage-sniffed connection (`copilot sniff: upstream dial failed`, `crux/proxy/github_proxy_copilot.go`) — typically DNS or network unreachability of an agent-requested host (for example a dial timeout to `telemetry.enterprise.githubcopilot.com`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `proxy_plain_forward_failed` | the proxy could not complete an agent's plain-HTTP upstream request (`proxy: plain HTTP forward failed`, `crux/proxy/github_proxy_connect.go`), returning HTTP 502. Check the requested host and logged error, including DNS and outbound connectivity; this differs from a destination refused by the proxy policy. Log-fed: ≥5 occurrences in 10 min, clears after 20 quiet minutes | medium | diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index e0db4f2f06..1832d03f99 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -102,6 +102,11 @@ const ( ClassTranscriptScanSlow = "transcript_scan_slow" ClassSharedCIRerunForbidden = "shared_ci_rerun_forbidden" ClassIssueClaimEffectFailed = "issue_claim_effect_failed" + // ClassSpekStreamUnavailable: a Spektacular live meeting child + // (`spektacular meeting`) keeps exiting and is restarted with backoff + // (hivecommons/hive#18630). A spektacular without the command is + // detected once and never restarted, so it does not reach this class. + ClassSpekStreamUnavailable = "spek_stream_unavailable" // A restart or normal stop can invalidate a pending bootstrap without // indicating a restart storm; only repeated interruptions are reportable. ClassStartupKickInterrupted = "startup_kick_interrupted" @@ -228,6 +233,7 @@ var Classes = map[string]ClassSpec{ ClassAgentLifecycleRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to send kick", "agent cli crashed (bare shell detected)", "tmux kill-session before restart failed", "agent tmux session missing", "would restart dead agent"}}, ClassSharedCIRerunForbidden: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"shared-ci rerun forbidden"}}, ClassIssueClaimEffectFailed: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue-claims: github side effect failed"}}, + ClassSpekStreamUnavailable: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"[spektacular] stream child exited; restarting", "[spektacular] stream child keeps exiting; giving up", "[spektacular] stream child did not restart"}}, ClassTranscriptScanSlow: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"transcript activity scan outlived its interval"}}, } diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index 3e7b49b7c4..4362223367 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -452,6 +452,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassSharedCIRerunForbidden, message: "shared-ci rerun forbidden: the App installation lacks Actions write permission for rerun-failed-jobs"}, {class: ClassIssueClaimEffectFailed, message: "issue-claims: GitHub side effect failed"}, {class: ClassTranscriptScanSlow, message: "copilot transcript activity scan outlived its interval; agent liveness reads may lag"}, + {class: ClassSpekStreamUnavailable, message: "[spektacular] stream child exited; restarting"}, {class: ClassBeadFactIngestFailed, message: "failed to ingest bead fact"}, {class: ClassSelfReportClearedFailed, message: "fleet report: cleared comment failed"}, {class: ClassGitHubAppRepoNotCovered, message: "github app recheck: installation does not cover every configured repo"}, @@ -1273,6 +1274,8 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { "ioscan classifier failed closed; text withheld": ClassIOScanClassifierUnavail, "watchdog: agent credential failure": ClassAgentLoginRequired, "ioscan classifier failed open; text passes on unclassified": ClassIOScanClassifierUnavail, + "[spektacular] stream child exited; restarting": ClassSpekStreamUnavailable, + "[spektacular] stream child keeps exiting; giving up": ClassSpekStreamUnavailable, "ioscan fail-closed blocked kick": ClassIOScanClassifierUnavail, "ioscan fail-closed blocked held-PR coordination kick": ClassIOScanClassifierUnavail, "proxy client TLS handshake failed": ClassProxyTLSHandshakeFailed, diff --git a/src/pkg/meetings/live/session.go b/src/pkg/meetings/live/session.go index 36ecaad3da..f52d1618d7 100644 --- a/src/pkg/meetings/live/session.go +++ b/src/pkg/meetings/live/session.go @@ -36,6 +36,14 @@ const ( spekPausedStatus = "Notes and clarifying questions are paused: Spektacular live mode is not running right now (restarting, or it needs a spektacular with the meeting command, hivecommons/spektacular#126). The transcript, the persona and research still work." ) +// spekUnsupported reports whether spek says the installed spektacular has no +// live meeting command (spektacular.Stream.Unsupported). A Spek without that +// method is never unsupported. +func spekUnsupported(spek Spek) bool { + u, ok := spek.(interface{ Unsupported() bool }) + return ok && u.Unsupported() +} + // setSpekStatus updates the panel's Spektacular status when it changes. func (s *session) setSpekStatus(status string) { if s.state.SpekStatus == status { @@ -69,6 +77,11 @@ type session struct { spek Spek state State rate rateLimiter + // spekGone is set once Spek is down for good for this meeting (its + // lines closed, or the installed spektacular has no meeting command): + // nothing more is forwarded to it, so a meeting without live mode does + // not log a warning per transcript line (hivecommons/hive#18630). + spekGone bool // persona is the meeting's attendee persona, nil when it is off. With // it on, it owns the bot's voice in the meeting: Spek's say intents are // dropped, its chat posts are offered to the persona, and its @@ -159,7 +172,13 @@ func (s *session) save() { func (s *session) run(ctx context.Context) { defer close(s.done) lines := s.spek.Lines() - ending, spekGone := false, false + ending := false + if spekUnsupported(s.spek) { + // The installed spektacular has no `meeting` command: live mode is + // off from the start; the stream already logged it once per binary. + s.spekGone, lines = true, nil + s.setSpekStatus(spekStoppedStatus) + } var replies <-chan betty.Reply if s.persona != nil { s.persona.Start(ctx) @@ -183,7 +202,7 @@ func (s *session) run(ctx context.Context) { if u.Meeting.State.Terminal() { ending = true close(s.ended) - if spekGone { + if s.spekGone { s.save() return } @@ -203,8 +222,12 @@ func (s *session) run(ctx context.Context) { // while the meeting goes on: keep the session for the panel, // research signals and approvals; only Spek's questions and // notes stop. - spekGone, lines = true, nil - s.log().Warn("[meetings] spektacular live mode stopped; continuing without it") + s.spekGone, lines = true, nil + if spekUnsupported(s.spek) { + s.log().Info("[meetings] spektacular live mode unavailable; continuing without it") + } else { + s.log().Warn("[meetings] spektacular live mode stopped; continuing without it") + } s.setSpekStatus(spekStoppedStatus) continue } @@ -255,13 +278,14 @@ func (s *session) onUpdate(u meetings.Update) { default: return } - if len(ev.Raw) > 0 { + if len(ev.Raw) > 0 && !s.spekGone { if err := s.spek.SendLine(ev.Raw); err != nil { - s.log().Warn("[meetings] forwarding to spek", "error", err) // The child is down (between restarts, or a spektacular // without the meeting command): say so on the panel now rather - // than only once it gives up. + // than only once it gives up. Warn on that transition only, not + // on every line while it stays down. if s.state.SpekStatus == "" { + s.log().Warn("[meetings] forwarding to spek", "error", err) s.setSpekStatus(spekPausedStatus) } } @@ -297,6 +321,9 @@ func (s *session) matchAnswer(ev *meetings.Event) { } a.Answer, a.AnsweredBy, a.AnsweredAt = ev.Text, ev.Who(), &now answered, changed = true, true + if s.spekGone { + continue + } if err := s.spek.Send(map[string]string{"type": "answer", "qid": a.QID, "text": ev.Text, "by": ev.Who()}); err != nil { s.log().Warn("[meetings] sending answer to spek", "error", err) } diff --git a/src/pkg/meetings/live/spek_unavailable_test.go b/src/pkg/meetings/live/spek_unavailable_test.go new file mode 100644 index 0000000000..bbc1432819 --- /dev/null +++ b/src/pkg/meetings/live/spek_unavailable_test.go @@ -0,0 +1,95 @@ +package live + +import ( + "bytes" + "errors" + "log/slog" + "strings" + "sync" + "testing" + + "github.com/hivecommons/hive/pkg/meetings" +) + +// syncLog is a goroutine-safe log sink. +type syncLog struct { + mu sync.Mutex + b bytes.Buffer +} + +func (l *syncLog) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.b.Write(p) +} + +func (l *syncLog) String() string { + l.mu.Lock() + defer l.mu.Unlock() + return l.b.String() +} + +// end ends the meeting and waits for its session to finish, so every +// update pushed before it has been applied. +func (h *harness) end() { + h.t.Helper() + ended := h.m + ended.State = meetings.StateEnded + h.mt.push(meetings.Update{Meeting: ended}) + h.eventually("session gone", func() bool { _, ok := h.r.session(h.m.ID); return !ok }) +} + +// unsupportedSpek is a Spek for a spektacular without the meeting command: +// it is done from the start and says so (spektacular.Stream.Unsupported). +type unsupportedSpek struct{ *fakeSpek } + +func (unsupportedSpek) Unsupported() bool { return true } + +// TestSpekUnsupportedNoForwardingNoWarnings pins hivecommons/hive#18630: a +// spektacular without `meeting` turns live mode off for the meeting at once +// (the panel says notes and questions are off), and transcript lines are +// not forwarded or warned about one by one. +func TestSpekUnsupportedNoForwardingNoWarnings(t *testing.T) { + logs := &syncLog{} + spek := unsupportedSpek{newFakeSpek()} + spek.Close(nil) // done from the start: Lines() is closed + h := newHarness(t, Policy{}, func(o *Options) { + o.NewSpek = func(meetings.Meeting) Spek { return spek } + o.Logger = slog.New(slog.NewTextHandler(logs, nil)) + }) + h.live() + h.eventually("stopped status", func() bool { + return strings.HasPrefix(h.state().SpekStatus, "Notes and clarifying questions are off for this meeting") + }) + h.line(meetings.EventSegment, "Ana", "let's start", true) + h.line(meetings.EventChat, "Bo", "agenda?", true) + h.line(meetings.EventSegment, "Ana", "first item", true) + h.end() + if got := spek.stdin(); len(got) != 0 { + t.Fatalf("lines forwarded to an unsupported spek: %v", got) + } + if out := logs.String(); strings.Contains(out, "forwarding to spek") || strings.Contains(out, "level=WARN") { + t.Fatalf("unexpected warnings: %s", out) + } +} + +// TestSpekForwardingWarnsOncePerOutage pins that a child that is down +// between restarts warns on the first failed forward, not on every line. +func TestSpekForwardingWarnsOncePerOutage(t *testing.T) { + logs := &syncLog{} + h := newHarness(t, Policy{}, func(o *Options) { o.Logger = slog.New(slog.NewTextHandler(logs, nil)) }) + h.live() + h.spek.mu.Lock() + h.spek.sendErr = errors.New("spektacular: stream child is not running") + h.spek.mu.Unlock() + for _, text := range []string{"one", "two", "three"} { + h.line(meetings.EventSegment, "Ana", text, true) + } + h.eventually("paused status", func() bool { + return strings.HasPrefix(h.state().SpekStatus, "Notes and clarifying questions are paused") + }) + h.end() + if n := strings.Count(logs.String(), "forwarding to spek"); n != 1 { + t.Fatalf("forwarding warnings = %d, want 1:\n%s", n, logs.String()) + } +} diff --git a/src/pkg/spektacular/stream.go b/src/pkg/spektacular/stream.go index 1c3f4f68c3..88f3330fab 100644 --- a/src/pkg/spektacular/stream.go +++ b/src/pkg/spektacular/stream.go @@ -9,7 +9,9 @@ import ( "fmt" "io" "log/slog" + "os" "os/exec" + "strings" "sync" "time" ) @@ -30,10 +32,35 @@ const ( maxStreamLine = 4 << 20 // streamLineBuffer is the stdout line channel depth. streamLineBuffer = 256 + // DefaultStreamProbeTimeout bounds the one-time capability probe. + DefaultStreamProbeTimeout = 10 * time.Second + // maxExitOutput bounds the stdout kept from a child to read its exit + // reason (the CLI's JSON error envelope is a few hundred bytes). + maxExitOutput = 4 << 10 + // unknownCommandMarker is how the CLI (cobra, wrapped in its JSON + // error envelope) says the verb does not exist in this build, for + // example `unknown command "meeting" for "spektacular"`. + unknownCommandMarker = "unknown command" ) -// ErrStreamNotRunning is returned by Send when no child is running. -var ErrStreamNotRunning = errors.New("spektacular: stream child is not running") +var ( + // ErrStreamNotRunning is returned by Send when no child is running. + ErrStreamNotRunning = errors.New("spektacular: stream child is not running") + // ErrStreamUnsupported is Err() for a stream whose verb the installed + // spektacular does not have (a release without `meeting`, + // hivecommons/spektacular#126). Such a stream never starts or restarts + // a child: Lines() is closed at once and Unsupported reports true. + ErrStreamUnsupported = errors.New("spektacular: the installed spektacular has no such command") +) + +// probeCache remembers, per binary file and probe, whether the verb exists, +// so the probe runs once per installed binary rather than once per stream. +// Keys are probeKey values; values are bool (true means supported). +var probeCache sync.Map + +// unsupportedLogged remembers which binaries were already logged as lacking +// the verb, so the INFO line is written once per binary, not per meeting. +var unsupportedLogged sync.Map // StreamOptions tunes a Stream. Zero values use the defaults. type StreamOptions struct { @@ -47,6 +74,13 @@ type StreamOptions struct { OnRestart func() // Env is the child environment; nil inherits hive's. Env []string + // Probe, when set, is a cheap command line (for example + // `meeting --help`) Start runs once per installed binary before the + // first child. When the CLI answers that the command is unknown, the + // stream is unsupported: no child is started or restarted. + Probe []string + // ProbeTimeout bounds the probe. + ProbeTimeout time.Duration } // Stream supervises one long-lived child. Lines() carries stdout lines @@ -66,6 +100,9 @@ type Stream struct { started bool restarts int exitErr error + // lastWarn is when the restart WARN was last written; restarts inside + // one BackoffMax window after it log at INFO. + lastWarn time.Time } func newStream(binary string, args []string, opts StreamOptions) *Stream { @@ -81,6 +118,9 @@ func newStream(binary string, args []string, opts StreamOptions) *Stream { if opts.CloseGrace <= 0 { opts.CloseGrace = DefaultStreamCloseGrace } + if opts.ProbeTimeout <= 0 { + opts.ProbeTimeout = DefaultStreamProbeTimeout + } log := opts.Logger if log == nil { log = slog.New(slog.DiscardHandler) @@ -111,8 +151,15 @@ func (s *Stream) Err() error { return s.exitErr } +// Unsupported reports whether the installed spektacular lacks the stream's +// verb, so the stream ended without (re)starting a child. +func (s *Stream) Unsupported() bool { return errors.Is(s.Err(), ErrStreamUnsupported) } + // Start launches the first child and the supervision loop. It fails when -// the first child cannot be started at all. +// the first child cannot be started at all. A stream whose verb the +// installed spektacular lacks (see StreamOptions.Probe) is not an error: +// Start returns nil with the stream already done and Unsupported true, so +// the caller carries on without it. func (s *Stream) Start(ctx context.Context) error { s.mu.Lock() if s.started { @@ -121,6 +168,12 @@ func (s *Stream) Start(ctx context.Context) error { } s.started = true s.mu.Unlock() + if !s.supported(ctx) { + s.finishUnsupported() + close(s.lines) + close(s.done) + return nil + } cmd, out, err := s.spawn(ctx) if err != nil { close(s.lines) @@ -131,6 +184,63 @@ func (s *Stream) Start(ctx context.Context) error { return nil } +// finishUnsupported ends a stream whose verb does not exist, before or +// instead of any (re)start. +func (s *Stream) finishUnsupported() { + s.mu.Lock() + s.exitErr = ErrStreamUnsupported + s.mu.Unlock() + if _, seen := unsupportedLogged.LoadOrStore(s.binary, true); !seen { + s.log.Info("[spektacular] live mode unavailable: the installed spektacular has no such command; not starting it", + "binary", s.binary, "args", strings.Join(s.args, " ")) + } +} + +// probeKey identifies one installed binary file and probe command line, so +// a replaced binary is probed again. +func probeKey(path string, fi os.FileInfo, probe []string) string { + return fmt.Sprintf("%s|%d|%d|%s", path, fi.Size(), fi.ModTime().UnixNano(), strings.Join(probe, "\x00")) +} + +// supported runs the probe (once per binary) and reports whether the verb +// exists. Anything inconclusive (no probe, binary not found, probe timeout +// or a failure that is not "unknown command") counts as supported, so the +// normal start path and its errors apply. +func (s *Stream) supported(ctx context.Context) bool { + if len(s.opts.Probe) == 0 { + return true + } + path, err := exec.LookPath(s.binary) + if err != nil { + return true + } + fi, err := os.Stat(path) + if err != nil { + return true + } + key := probeKey(path, fi, s.opts.Probe) + if v, cached := probeCache.Load(key); cached { + known, _ := v.(bool) + return known + } + pctx, cancel := context.WithTimeout(ctx, s.opts.ProbeTimeout) + defer cancel() + cmd := exec.CommandContext(pctx, path, s.opts.Probe...) + cmd.Env = s.opts.Env + out, runErr := cmd.CombinedOutput() + if pctx.Err() != nil { + return true + } + ok := runErr == nil || !isUnknownCommand(out) + probeCache.Store(key, ok) + return ok +} + +// isUnknownCommand reports whether CLI output says the verb does not exist. +func isUnknownCommand(out []byte) bool { + return bytes.Contains(bytes.ToLower(out), []byte(unknownCommandMarker)) +} + func (s *Stream) spawn(ctx context.Context) (*exec.Cmd, io.ReadCloser, error) { cmd := exec.Command(s.binary, s.args...) cmd.Env = s.opts.Env @@ -160,7 +270,8 @@ func (s *Stream) loop(ctx context.Context, cmd *exec.Cmd, out io.ReadCloser) { defer close(s.lines) backoff := s.opts.BackoffMin for { - err := s.pump(ctx, cmd, out) + head := &headBuffer{max: maxExitOutput} + err := s.pump(ctx, cmd, out, head) s.mu.Lock() s.stdin, s.cmd, s.exitErr = nil, nil, err closing := s.closing @@ -169,11 +280,17 @@ func (s *Stream) loop(ctx context.Context, cmd *exec.Cmd, out io.ReadCloser) { if closing || ctx.Err() != nil || err == nil { return } + // The verb does not exist in this build (no probe was configured, + // or the binary changed under it): restarting cannot help. + if isUnknownCommand(head.b) { + s.finishUnsupported() + return + } if restarts >= s.opts.MaxRestarts { s.log.Error("[spektacular] stream child keeps exiting; giving up", "binary", s.binary, "restarts", restarts, "error", err) return } - s.log.Warn("[spektacular] stream child exited; restarting", "error", err, "backoff", backoff.String()) + s.logRestart(err, backoff, restarts) select { case <-ctx.Done(): return @@ -201,9 +318,42 @@ func (s *Stream) loop(ctx context.Context, cmd *exec.Cmd, out io.ReadCloser) { } } +// logRestart writes the restart line at WARN at most once per BackoffMax +// window and at INFO otherwise, so one child's backoff series is a single +// warning rather than one per attempt (hivecommons/hive#18630). +func (s *Stream) logRestart(err error, backoff time.Duration, restarts int) { + now := time.Now() + s.mu.Lock() + warn := s.lastWarn.IsZero() || now.Sub(s.lastWarn) >= s.opts.BackoffMax + if warn { + s.lastWarn = now + } + s.mu.Unlock() + level := slog.LevelInfo + if warn { + level = slog.LevelWarn + } + s.log.Log(context.Background(), level, "[spektacular] stream child exited; restarting", + "error", err, "backoff", backoff.String(), "restarts", restarts) +} + +// headBuffer keeps the first max bytes of a child's stdout. +type headBuffer struct { + max int + b []byte +} + +func (h *headBuffer) add(line []byte) { + if room := h.max - len(h.b); room > 0 { + h.b = append(h.b, line[:min(len(line), room)]...) + h.b = append(h.b, '\n') + } +} + // pump copies stdout lines until the child exits, killing it when ctx -// ends. It returns the child's exit error. -func (s *Stream) pump(ctx context.Context, cmd *exec.Cmd, out io.Reader) error { +// ends. It returns the child's exit error. head, when set, keeps the start +// of stdout so the caller can read the exit reason. +func (s *Stream) pump(ctx context.Context, cmd *exec.Cmd, out io.Reader, head *headBuffer) error { readDone := make(chan struct{}) go func() { defer close(readDone) @@ -214,6 +364,9 @@ func (s *Stream) pump(ctx context.Context, cmd *exec.Cmd, out io.Reader) error { if len(line) == 0 { continue } + if head != nil { + head.add(line) + } s.lines <- append([]byte(nil), line...) } }() diff --git a/src/pkg/spektacular/stream_test.go b/src/pkg/spektacular/stream_test.go index 4755398d85..b2e859cc68 100644 --- a/src/pkg/spektacular/stream_test.go +++ b/src/pkg/spektacular/stream_test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "log/slog" + "os" "path/filepath" "strings" "sync" @@ -184,3 +185,151 @@ func TestStreamLogWriter(t *testing.T) { t.Fatalf("logs %q buf %d", logs.String(), len(w.buf)) } } + +// fakeCLI writes an executable shell script standing in for spektacular and +// returns its path and a file counting its invocations. +func fakeCLI(t *testing.T, body string) (bin, calls string) { + t.Helper() + dir := t.TempDir() + bin, calls = filepath.Join(dir, "spektacular"), filepath.Join(dir, "calls") + script := "#!/bin/sh\necho x >> '" + calls + "'\n" + body + "\n" + if err := os.WriteFile(bin, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + return bin, calls +} + +func callCount(t *testing.T, calls string) int { + t.Helper() + b, err := os.ReadFile(calls) + if errors.Is(err, os.ErrNotExist) { + return 0 + } + if err != nil { + t.Fatal(err) + } + return strings.Count(string(b), "x\n") +} + +// oldCLIBody is what a spektacular release without `meeting` prints for it: +// cobra's unknown-command error in the CLI's indented JSON error envelope on +// stdout, exit 1 (hivecommons/hive#18630). +const oldCLIBody = `cat <<'JSON' +{ + "error": true, + "code": "internal_error", + "message": "unknown command \"meeting\" for \"spektacular\"", + "next_action": "" +} +JSON +exit 1` + +// TestStreamProbeUnsupportedNeverStartsChild pins #18630: when the probe +// says the verb is unknown, no child is started or restarted, the stream is +// done and Unsupported, and the probe runs once per installed binary. +func TestStreamProbeUnsupportedNeverStartsChild(t *testing.T) { + bin, calls := fakeCLI(t, oldCLIBody) + logs := &lockedLog{} + opts := StreamOptions{ + Logger: slog.New(slog.NewTextHandler(logs, nil)), Probe: []string{"meeting", "--help"}, + BackoffMin: time.Millisecond, + } + for i := 0; i < 2; i++ { + s := StreamExec(bin, []string{"meeting", "--from", "stdin"}, opts) + if err := s.Start(context.Background()); err != nil { + t.Fatalf("Start = %v, want nil for an unsupported verb", err) + } + waitDone(t, s) + if _, ok := <-s.Lines(); ok { + t.Fatal("lines must be closed") + } + if !s.Unsupported() || !errors.Is(s.Err(), ErrStreamUnsupported) || s.Restarts() != 0 { + t.Fatalf("unsupported=%v err=%v restarts=%d", s.Unsupported(), s.Err(), s.Restarts()) + } + if !errors.Is(s.SendLine([]byte("{}")), ErrStreamNotRunning) { + t.Fatal("Send must fail") + } + } + if n := callCount(t, calls); n != 1 { + t.Fatalf("binary ran %d times, want only the one probe", n) + } + out := logs.String() + if strings.Contains(out, "level=WARN") || strings.Contains(out, "level=ERROR") || strings.Count(out, "live mode unavailable") != 1 { + t.Fatalf("want one INFO line and no warnings:\n%s", out) + } +} + +// TestStreamUnknownCommandExitDoesNotRestart covers a stream without a +// probe: the child's exit reason says the verb is unknown, so it is not +// restarted. +func TestStreamUnknownCommandExitDoesNotRestart(t *testing.T) { + bin, calls := fakeCLI(t, oldCLIBody) + s := StreamExec(bin, []string{"meeting"}, StreamOptions{BackoffMin: time.Millisecond, MaxRestarts: 3}) + if err := s.Start(context.Background()); err != nil { + t.Fatal(err) + } + for range s.Lines() { + } + waitDone(t, s) + if !s.Unsupported() || s.Restarts() != 0 || callCount(t, calls) != 1 { + t.Fatalf("unsupported=%v restarts=%d calls=%d", s.Unsupported(), s.Restarts(), callCount(t, calls)) + } +} + +// TestStreamProbeSupportedStarts: a binary that knows the verb starts as +// before. +func TestStreamProbeSupportedStarts(t *testing.T) { + bin, calls := fakeCLI(t, `[ "$2" = "--help" ] && exit 0 +while read -r l; do echo "$l"; done`) + s := StreamExec(bin, []string{"meeting"}, StreamOptions{Probe: []string{"meeting", "--help"}}) + if err := s.Start(context.Background()); err != nil { + t.Fatal(err) + } + if err := s.SendLine([]byte(`{"type":"segment"}`)); err != nil { + t.Fatal(err) + } + if got := readLine(t, s); got != `{"type":"segment"}` { + t.Fatalf("echo = %q", got) + } + s.Close(nil) + waitDone(t, s) + if s.Unsupported() || callCount(t, calls) != 2 { + t.Fatalf("unsupported=%v calls=%d", s.Unsupported(), callCount(t, calls)) + } +} + +// TestStreamRestartWarnsOncePerBackoffWindow pins #18630: a child that keeps +// exiting is restarted with backoff, but the restart line is a WARN at most +// once per BackoffMax window (INFO otherwise), so one backoff series does not +// look like a repeated unclassified error. +func TestStreamRestartWarnsOncePerBackoffWindow(t *testing.T) { + logs := &lockedLog{} + s := StreamExec("sh", []string{"-c", "exit 3"}, StreamOptions{ + Logger: slog.New(slog.NewTextHandler(logs, nil)), + BackoffMin: time.Millisecond, BackoffMax: time.Hour, MaxRestarts: 4, + }) + if err := s.Start(context.Background()); err != nil { + t.Fatal(err) + } + for range s.Lines() { + } + waitDone(t, s) + if s.Restarts() != 4 { + t.Fatalf("restarts = %d", s.Restarts()) + } + warn, info := 0, 0 + for _, l := range strings.Split(logs.String(), "\n") { + if !strings.Contains(l, "stream child exited; restarting") { + continue + } + switch { + case strings.Contains(l, "level=WARN"): + warn++ + case strings.Contains(l, "level=INFO"): + info++ + } + } + if warn != 1 || info != 3 { + t.Fatalf("warn=%d info=%d, want 1 and 3:\n%s", warn, info, logs.String()) + } +}