Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-18630-spek-stream-restart-loop.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Meetings no longer restart-loop the Spektacular live child when the installed spektacular has no `meeting` command (before hivecommons/spektacular#126): hive probes `spektacular meeting --help` once per installed binary, keeps live mode off without starting the child, stops forwarding transcript lines to it, and the Meetings panel still says notes and clarifying questions are off. A supported child that keeps exiting still restarts with capped exponential backoff, warns at most once per backoff window, and is self-reported as `spek_stream_unavailable` instead of `unclassified_error_repeated` ([#18630](https://github.com/hivecommons/hive/issues/18630)).
10 changes: 9 additions & 1 deletion src/cmd/hive/meetingswire.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,12 @@ func wireMeetingsManager(cfg *config.Config, sup quorumSupervisor, logger *slog.
return mgr, store
}

// meetingSpekProbeArgs checks once per installed binary that spektacular
// has the live `meeting` command (hivecommons/spektacular#126); a release
// without it answers "unknown command" and live mode stays off without a
// child restart loop (hivecommons/hive#18630).
var meetingSpekProbeArgs = []string{"meeting", "--help"}

// meetingSpekArgs is the contract's live-mode command line
// (quorum.contract/v1 section 2).
func meetingSpekArgs(m meetings.Meeting, contextDir string) []string {
Expand Down Expand Up @@ -205,7 +211,9 @@ func wireMeetingsLive(cfg *config.Config, mgr *meetings.Manager, store *meetings
Store: store,
NewSpek: func(mt meetings.Meeting) live.Spek {
env := meetingSpekEnv(os.Getenv, llm, hiveBin, meetingUsageFile(store, mt.ID), slotsDir, maxTokens)
return spektacular.StreamExec(binary, meetingSpekArgs(mt, filepath.Join(store.Dir(), mt.ID)), spektacular.StreamOptions{Logger: logger, Env: env})
return spektacular.StreamExec(binary, meetingSpekArgs(mt, filepath.Join(store.Dir(), mt.ID)), spektacular.StreamOptions{
Logger: logger, Env: env, Probe: meetingSpekProbeArgs,
})
},
SpekMode: llm.Label(),
UsageFile: func(id string) string {
Expand Down
1 change: 1 addition & 0 deletions src/docs/fleet-report.md
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ title is `[selfreport] <class>: <short message>`.
| `github_fetch_transient` | a non-critical GitHub fetch failed and the feature degrades gracefully (`channel commit date: fetch failed — the row will render without a timestamp`, `pkg/hub/channel_commit_date.go`) — typically a rate limit or transient API error, not a hive defect. Reported only when persistent: log-fed, same ≥5-in-10-min repeat threshold and idle clear as `agent_pane_timeout`. The sibling `SHA poll: commit message fetch non-200` has its own class, `sha_poll_commit_message_unavailable` | low |
| `label_provisioning_failed` | provisioning the issue stage labels on a repo failed (`issue stage labels: provisioning failed`, `pkg/github/issue_stage_labels.go`) — typically the App lacking label permission on that repo or a transient API error; diagnostics carry the repo and scrubbed error. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low |
| `agent_cli_restart_loop` | the agent CLI crashed (bare shell) or sat on a consent screen and the harness restarted it before the kick (`agent CLI crashed or stuck on consent screen, restarting before kick`, `crux/agent/manager_kick.go`, `kick_async.go`), repeated ≥5 times in 10 min — a restart loop, not transient noise. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium |
| `spek_stream_unavailable` | a Spektacular live meeting child (`spektacular meeting`, `pkg/spektacular/stream.go`) keeps exiting and is restarted with exponential backoff (`[spektacular] stream child exited; restarting`), or gave up after its restart cap (`[spektacular] stream child keeps exiting; giving up`). The restart line is a WARN at most once per backoff window and INFO otherwise. A spektacular release without the `meeting` command (before hivecommons/spektacular#126) is not this class: the stream probes `spektacular meeting --help` once per installed binary, logs one INFO and never starts or restarts the child, and the Meetings panel says notes and clarifying questions are off ([#18630](https://github.com/hivecommons/hive/issues/18630)). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low |
| `copilot_cli_hung` | a Copilot agent showed none of the CLI's startup chrome (input prompt, help hint, login request) 180 s after launch, outside kick delivery, so the hang detector killed its session, ran bare `copilot` to capture the error and relaunched the agent (`copilot hung with no CLI prompt, running diagnostic`, `crux/agent/manager_poll.go`), or that diagnostic saw no output before its own deadline and relaunched anyway (`diagnostic: timed out waiting for copilot error output`, `crux/agent/manager_credentials.go`; the line carries the pane's `output_tail`). The key is the message, so ≥5 in 10 min counts every Copilot agent on the hive together: its Copilot agents keep failing to reach their prompt. Typical causes are a stale Copilot token the CLI hangs on (the diagnostic restores or clears it when it sees an auth error), a blocked network path to the Copilot API, or a CLI build whose startup screen the detector does not recognise (`cliReadyIndicators`, `crux/agent/pane_classify.go`). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium |
| `proxy_connect_dial_failed` | the hive proxy could not dial the upstream host of a CONNECT request (`proxy: CONNECT dial failed`, `crux/proxy/github_proxy_connect.go`) or of a Copilot usage-sniffed connection (`copilot sniff: upstream dial failed`, `crux/proxy/github_proxy_copilot.go`) — typically DNS or network unreachability of an agent-requested host (for example a dial timeout to `telemetry.enterprise.githubcopilot.com`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium |
| `proxy_plain_forward_failed` | the proxy could not complete an agent's plain-HTTP upstream request (`proxy: plain HTTP forward failed`, `crux/proxy/github_proxy_connect.go`), returning HTTP 502. Check the requested host and logged error, including DNS and outbound connectivity; this differs from a destination refused by the proxy policy. Log-fed: ≥5 occurrences in 10 min, clears after 20 quiet minutes | medium |
Expand Down
6 changes: 6 additions & 0 deletions src/pkg/fleetreport/runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,11 @@ const (
ClassTranscriptScanSlow = "transcript_scan_slow"
ClassSharedCIRerunForbidden = "shared_ci_rerun_forbidden"
ClassIssueClaimEffectFailed = "issue_claim_effect_failed"
// ClassSpekStreamUnavailable: a Spektacular live meeting child
// (`spektacular meeting`) keeps exiting and is restarted with backoff
// (hivecommons/hive#18630). A spektacular without the command is
// detected once and never restarted, so it does not reach this class.
ClassSpekStreamUnavailable = "spek_stream_unavailable"
// A restart or normal stop can invalidate a pending bootstrap without
// indicating a restart storm; only repeated interruptions are reportable.
ClassStartupKickInterrupted = "startup_kick_interrupted"
Expand Down Expand Up @@ -228,6 +233,7 @@ var Classes = map[string]ClassSpec{
ClassAgentLifecycleRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to send kick", "agent cli crashed (bare shell detected)", "tmux kill-session before restart failed", "agent tmux session missing", "would restart dead agent"}},
ClassSharedCIRerunForbidden: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"shared-ci rerun forbidden"}},
ClassIssueClaimEffectFailed: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue-claims: github side effect failed"}},
ClassSpekStreamUnavailable: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"[spektacular] stream child exited; restarting", "[spektacular] stream child keeps exiting; giving up", "[spektacular] stream child did not restart"}},
ClassTranscriptScanSlow: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"transcript activity scan outlived its interval"}},
}

Expand Down
3 changes: 3 additions & 0 deletions src/pkg/fleetreport/runtime_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) {
{class: ClassSharedCIRerunForbidden, message: "shared-ci rerun forbidden: the App installation lacks Actions write permission for rerun-failed-jobs"},
{class: ClassIssueClaimEffectFailed, message: "issue-claims: GitHub side effect failed"},
{class: ClassTranscriptScanSlow, message: "copilot transcript activity scan outlived its interval; agent liveness reads may lag"},
{class: ClassSpekStreamUnavailable, message: "[spektacular] stream child exited; restarting"},
{class: ClassBeadFactIngestFailed, message: "failed to ingest bead fact"},
{class: ClassSelfReportClearedFailed, message: "fleet report: cleared comment failed"},
{class: ClassGitHubAppRepoNotCovered, message: "github app recheck: installation does not cover every configured repo"},
Expand Down Expand Up @@ -1273,6 +1274,8 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) {
"ioscan classifier failed closed; text withheld": ClassIOScanClassifierUnavail,
"watchdog: agent credential failure": ClassAgentLoginRequired,
"ioscan classifier failed open; text passes on unclassified": ClassIOScanClassifierUnavail,
"[spektacular] stream child exited; restarting": ClassSpekStreamUnavailable,
"[spektacular] stream child keeps exiting; giving up": ClassSpekStreamUnavailable,
"ioscan fail-closed blocked kick": ClassIOScanClassifierUnavail,
"ioscan fail-closed blocked held-PR coordination kick": ClassIOScanClassifierUnavail,
"proxy client TLS handshake failed": ClassProxyTLSHandshakeFailed,
Expand Down
41 changes: 34 additions & 7 deletions src/pkg/meetings/live/session.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,14 @@ const (
spekPausedStatus = "Notes and clarifying questions are paused: Spektacular live mode is not running right now (restarting, or it needs a spektacular with the meeting command, hivecommons/spektacular#126). The transcript, the persona and research still work."
)

// spekUnsupported reports whether spek says the installed spektacular has no
// live meeting command (spektacular.Stream.Unsupported). A Spek without that
// method is never unsupported.
func spekUnsupported(spek Spek) bool {
u, ok := spek.(interface{ Unsupported() bool })
return ok && u.Unsupported()
}

// setSpekStatus updates the panel's Spektacular status when it changes.
func (s *session) setSpekStatus(status string) {
if s.state.SpekStatus == status {
Expand Down Expand Up @@ -69,6 +77,11 @@ type session struct {
spek Spek
state State
rate rateLimiter
// spekGone is set once Spek is down for good for this meeting (its
// lines closed, or the installed spektacular has no meeting command):
// nothing more is forwarded to it, so a meeting without live mode does
// not log a warning per transcript line (hivecommons/hive#18630).
spekGone bool
// persona is the meeting's attendee persona, nil when it is off. With
// it on, it owns the bot's voice in the meeting: Spek's say intents are
// dropped, its chat posts are offered to the persona, and its
Expand Down Expand Up @@ -159,7 +172,13 @@ func (s *session) save() {
func (s *session) run(ctx context.Context) {
defer close(s.done)
lines := s.spek.Lines()
ending, spekGone := false, false
ending := false
if spekUnsupported(s.spek) {
// The installed spektacular has no `meeting` command: live mode is
// off from the start; the stream already logged it once per binary.
s.spekGone, lines = true, nil
s.setSpekStatus(spekStoppedStatus)
}
var replies <-chan betty.Reply
if s.persona != nil {
s.persona.Start(ctx)
Expand All @@ -183,7 +202,7 @@ func (s *session) run(ctx context.Context) {
if u.Meeting.State.Terminal() {
ending = true
close(s.ended)
if spekGone {
if s.spekGone {
s.save()
return
}
Expand All @@ -203,8 +222,12 @@ func (s *session) run(ctx context.Context) {
// while the meeting goes on: keep the session for the panel,
// research signals and approvals; only Spek's questions and
// notes stop.
spekGone, lines = true, nil
s.log().Warn("[meetings] spektacular live mode stopped; continuing without it")
s.spekGone, lines = true, nil
if spekUnsupported(s.spek) {
s.log().Info("[meetings] spektacular live mode unavailable; continuing without it")
} else {
s.log().Warn("[meetings] spektacular live mode stopped; continuing without it")
}
s.setSpekStatus(spekStoppedStatus)
continue
}
Expand Down Expand Up @@ -255,13 +278,14 @@ func (s *session) onUpdate(u meetings.Update) {
default:
return
}
if len(ev.Raw) > 0 {
if len(ev.Raw) > 0 && !s.spekGone {
if err := s.spek.SendLine(ev.Raw); err != nil {
s.log().Warn("[meetings] forwarding to spek", "error", err)
// The child is down (between restarts, or a spektacular
// without the meeting command): say so on the panel now rather
// than only once it gives up.
// than only once it gives up. Warn on that transition only, not
// on every line while it stays down.
if s.state.SpekStatus == "" {
s.log().Warn("[meetings] forwarding to spek", "error", err)
s.setSpekStatus(spekPausedStatus)
}
}
Expand Down Expand Up @@ -297,6 +321,9 @@ func (s *session) matchAnswer(ev *meetings.Event) {
}
a.Answer, a.AnsweredBy, a.AnsweredAt = ev.Text, ev.Who(), &now
answered, changed = true, true
if s.spekGone {
continue
}
if err := s.spek.Send(map[string]string{"type": "answer", "qid": a.QID, "text": ev.Text, "by": ev.Who()}); err != nil {
s.log().Warn("[meetings] sending answer to spek", "error", err)
}
Expand Down
95 changes: 95 additions & 0 deletions src/pkg/meetings/live/spek_unavailable_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package live

import (
"bytes"
"errors"
"log/slog"
"strings"
"sync"
"testing"

"github.com/hivecommons/hive/pkg/meetings"
)

// syncLog is a goroutine-safe log sink.
type syncLog struct {
mu sync.Mutex
b bytes.Buffer
}

func (l *syncLog) Write(p []byte) (int, error) {
l.mu.Lock()
defer l.mu.Unlock()
return l.b.Write(p)
}

func (l *syncLog) String() string {
l.mu.Lock()
defer l.mu.Unlock()
return l.b.String()
}

// end ends the meeting and waits for its session to finish, so every
// update pushed before it has been applied.
func (h *harness) end() {
h.t.Helper()
ended := h.m
ended.State = meetings.StateEnded
h.mt.push(meetings.Update{Meeting: ended})
h.eventually("session gone", func() bool { _, ok := h.r.session(h.m.ID); return !ok })
}

// unsupportedSpek is a Spek for a spektacular without the meeting command:
// it is done from the start and says so (spektacular.Stream.Unsupported).
type unsupportedSpek struct{ *fakeSpek }

func (unsupportedSpek) Unsupported() bool { return true }

// TestSpekUnsupportedNoForwardingNoWarnings pins hivecommons/hive#18630: a
// spektacular without `meeting` turns live mode off for the meeting at once
// (the panel says notes and questions are off), and transcript lines are
// not forwarded or warned about one by one.
func TestSpekUnsupportedNoForwardingNoWarnings(t *testing.T) {
logs := &syncLog{}
spek := unsupportedSpek{newFakeSpek()}
spek.Close(nil) // done from the start: Lines() is closed
h := newHarness(t, Policy{}, func(o *Options) {
o.NewSpek = func(meetings.Meeting) Spek { return spek }
o.Logger = slog.New(slog.NewTextHandler(logs, nil))
})
h.live()
h.eventually("stopped status", func() bool {
return strings.HasPrefix(h.state().SpekStatus, "Notes and clarifying questions are off for this meeting")
})
h.line(meetings.EventSegment, "Ana", "let's start", true)
h.line(meetings.EventChat, "Bo", "agenda?", true)
h.line(meetings.EventSegment, "Ana", "first item", true)
h.end()
if got := spek.stdin(); len(got) != 0 {
t.Fatalf("lines forwarded to an unsupported spek: %v", got)
}
if out := logs.String(); strings.Contains(out, "forwarding to spek") || strings.Contains(out, "level=WARN") {
t.Fatalf("unexpected warnings: %s", out)
}
}

// TestSpekForwardingWarnsOncePerOutage pins that a child that is down
// between restarts warns on the first failed forward, not on every line.
func TestSpekForwardingWarnsOncePerOutage(t *testing.T) {
logs := &syncLog{}
h := newHarness(t, Policy{}, func(o *Options) { o.Logger = slog.New(slog.NewTextHandler(logs, nil)) })
h.live()
h.spek.mu.Lock()
h.spek.sendErr = errors.New("spektacular: stream child is not running")
h.spek.mu.Unlock()
for _, text := range []string{"one", "two", "three"} {
h.line(meetings.EventSegment, "Ana", text, true)
}
h.eventually("paused status", func() bool {
return strings.HasPrefix(h.state().SpekStatus, "Notes and clarifying questions are paused")
})
h.end()
if n := strings.Count(logs.String(), "forwarding to spek"); n != 1 {
t.Fatalf("forwarding warnings = %d, want 1:\n%s", n, logs.String())
}
}
Loading
Loading