diff --git a/changelog.d/fixed-watchdog-login-flash-relaunch.md b/changelog.d/fixed-watchdog-login-flash-relaunch.md new file mode 100644 index 0000000000..f0b5abdb99 --- /dev/null +++ b/changelog.d/fixed-watchdog-login-flash-relaunch.md @@ -0,0 +1 @@ +- Watchdog no longer pages `PaneShowsLogin` for the sub-second login flash a relaunched Copilot CLI paints while re-validating its token: boot grace is now dated from the most recent CLI (re)launch, and login chrome must persist for `governor.watchdog.liveness.login_streak` consecutive polls (default 3) before `auth-required` is raised. diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index a3a081ee6a..02f424ba34 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -427,9 +427,10 @@ type AgentProcess struct { // this as "Waiting on CI" instead of "Working" so a stalled-on-CI agent // no longer looks indistinguishable from one making progress. AwaitingCI bool - TransientNudges int // total transient-API-error nudges sent (surfaced to the dashboard) - launchGen int // increments per launch; stale deliverStartupKick goroutines check it and drop - lastInferKickMarks int // no-action watchdog: tool-marker count in pane+scrollback just after kick delivery + TransientNudges int // total transient-API-error nudges sent (surfaced to the dashboard) + launchGen int // increments per launch; stale deliverStartupKick goroutines check it and drop + lastCLILaunchAt time.Time // when the CLI was last typed into the pane; dates watchdog boot grace on relaunch (#17320) + lastInferKickMarks int // no-action watchdog: tool-marker count in pane+scrollback just after kick delivery ProviderErrorClass string ProviderErrorLine string ProviderErrorBackoffUntil time.Time diff --git a/src/crux/agent/manager_launch.go b/src/crux/agent/manager_launch.go index 31713b6d74..9e878fc5db 100644 --- a/src/crux/agent/manager_launch.go +++ b/src/crux/agent/manager_launch.go @@ -313,8 +313,7 @@ func (m *Manager) launchInTmux(ctx context.Context, agent *AgentProcess) error { agent.LastError = "" agent.copilotModelFallbackRequested = "" agent.lastLaunchFailureBanner = "" - agent.StartedAt = &now - agent.launchGen++ + m.markCLILaunchedLocked(agent, now) m.logger.Info("audit: agent started", "name", agent.Name, "backend", backend, diff --git a/src/crux/agent/watchdog_fleet.go b/src/crux/agent/watchdog_fleet.go index 7445ad2bf9..f0d840acc0 100644 --- a/src/crux/agent/watchdog_fleet.go +++ b/src/crux/agent/watchdog_fleet.go @@ -144,7 +144,9 @@ func (f WatchdogFleet) Observe(name string) (watchdog.Observation, error) { // StartedAt dates the current launch so the reconciler can suppress dead // verdicts during boot. Copied by value: the field is a pointer the - // manager mutates on relaunch. + // manager mutates on relaunch. LastLaunchAt is the most recent CLI spawn + // (markCLILaunchedLocked), which a restored or resumed session can leave + // StartedAt lagging behind. var startedAt time.Time providerClass := "" providerLine := "" @@ -152,6 +154,7 @@ func (f WatchdogFleet) Observe(name string) (watchdog.Observation, error) { if agent.StartedAt != nil { startedAt = *agent.StartedAt } + lastLaunchAt := agent.lastCLILaunchAt if f.M.providerErrorBackoffRemainingLocked(agent, time.Now()) > 0 { providerClass = agent.ProviderErrorClass providerLine = agent.ProviderErrorLine @@ -166,6 +169,7 @@ func (f WatchdogFleet) Observe(name string) (watchdog.Observation, error) { ShowsLoginPrompt: needsLogin || paneShowsLoginPrompt(strings.Split(pane, "\n")), LastChange: lastChange, StartedAt: startedAt, + LastLaunchAt: lastLaunchAt, AuthAvailable: authAvailable, AuthKnown: authKnown, CredentialProven: credentialProven, @@ -439,3 +443,13 @@ func newestMtimeBounded(root string, maxDepth int) (time.Time, bool) { func (m *Manager) WatchdogSource() watchdog.Source { return WatchdogFleet{M: m} } + +// markCLILaunchedLocked records a completed CLI launch into the pane: the +// agent's StartedAt, the launch generation stale startup-kick goroutines +// check, and the relaunch timestamp the watchdog dates its boot grace from +// (#17320). Caller holds m.mu. +func (m *Manager) markCLILaunchedLocked(agent *AgentProcess, now time.Time) { + agent.StartedAt = &now + agent.launchGen++ + agent.lastCLILaunchAt = now +} diff --git a/src/crux/config/watchdog.go b/src/crux/config/watchdog.go index 5307e622a1..4ec92f45a3 100644 --- a/src/crux/config/watchdog.go +++ b/src/crux/config/watchdog.go @@ -42,6 +42,12 @@ type WatchdogLivenessConfig struct { // window exists because a launching CLI legitimately passes through both // states. "" = 5m. ShellPromptAfter string `yaml:"shell_prompt_after,omitempty"` + // LoginStreak is how many consecutive watchdog polls must see login + // chrome before the pane is classified auth-required and an operator is + // paged; fewer sightings report Unknown. A relaunched CLI flashes login + // chrome while re-validating its token, so one sighting is not a logout. + // 0 = 3; 1 disables the guard. + LoginStreak int `yaml:"login_streak,omitempty"` } // WatchdogRestartConfig holds the CrashLoopBackOff-analog knobs. diff --git a/src/crux/watchdog/classify.go b/src/crux/watchdog/classify.go index 565a104aff..bb8c8fb81e 100644 --- a/src/crux/watchdog/classify.go +++ b/src/crux/watchdog/classify.go @@ -1,6 +1,7 @@ package watchdog import ( + "fmt" "strings" "time" ) @@ -67,6 +68,13 @@ type Observation struct { // agent spawns a second concurrent CLI, the race cliBootGraceSeconds was // added for on the manager side. StartedAt time.Time + // LastLaunchAt is when the CLI process was most recently (re)launched + // into the pane; zero when the manager has not recorded one. StartedAt + // dates the agent, not the process: a kick restart, memory-drain recycle + // or watchdog restart hours into an agent's life spawns a fresh CLI that + // repeats the whole startup handshake (login chrome included), so boot + // grace is measured from the later of the two. + LastLaunchAt time.Time // AuthAvailable/AuthKnown carry the manager's owner-aware per-agent // credential-file verdict (AgentAuthState, #4619/#4641): AuthKnown=false // means the probe could not determine an answer and the reconciler must @@ -208,9 +216,12 @@ func Classify(obs Observation, now time.Time, s Settings) Classification { // Boot grace outranks every dead verdict. A launching agent legitimately // has no tmux session, no pane, and no CLI marker for the first seconds of // its life; without this, the watchdog would restart it underneath itself - // and spawn a second concurrent CLI. An agent with no StartedAt cannot be - // dated, so it gets no grace rather than an unbounded one. - booting := !obs.StartedAt.IsZero() && now.Sub(obs.StartedAt) < s.BootGrace + // and spawn a second concurrent CLI. An agent with no launch timestamp + // cannot be dated, so it gets no grace rather than an unbounded one. The + // grace is dated from the most recent CLI launch, not the agent's first: + // a relaunched CLI boots exactly like a new one (#17320). + launchedAt := obs.lastLaunch() + booting := !launchedAt.IsZero() && now.Sub(launchedAt) < s.BootGrace if !obs.SessionExists { if booting { @@ -234,11 +245,16 @@ func Classify(obs Observation, now time.Time, s Settings) Classification { // branch. Observed on a hosted spoke: rolling the pod paged quality, // scanner and ci-maintainer 3 minutes in, one event each, never // repeated -- all three were authenticated and working the whole time. + // Observed again 22h into the same agents' lives when their CLIs were + // relaunched mid-life (#17320): the grace was dated from the agent's + // first start, so the relaunch flash got none. Hence lastLaunch above, + // and the reconciler's ApplyLoginStreak on top for the relaunches the + // manager cannot date (the CLI re-executing itself, for one). // // Unknown rather than Ready: a login screen is not health, and this // says only that it is too early to conclude anything. An agent that // genuinely needs a human still shows the same chrome once grace - // elapses and pages on the very next tick. + // elapses and pages once the streak is met. if booting { return Classification{ClassUnknown, "pane shows login chrome inside boot grace"} } @@ -290,3 +306,26 @@ func Classify(obs Observation, now time.Time, s Settings) Classification { } return Classification{ClassUnknown, "pane matched no known signature"} } + +// lastLaunch is the later of the agent's start and its most recent CLI +// relaunch — the instant boot grace is measured from. +func (o Observation) lastLaunch() time.Time { + if o.LastLaunchAt.After(o.StartedAt) { + return o.LastLaunchAt + } + return o.StartedAt +} + +// ApplyLoginStreak is the watchdog's counterpart to the pane poller's +// loginStreak guard: an auth-required verdict only stands once login chrome +// has been seen on `required` consecutive polls, `streak` being the count +// including this one. Below that it is downgraded to unknown — the honest +// verdict for a CLI that may simply be mid-handshake after a relaunch the +// manager could not date (#17320). Any other class passes through untouched; +// a required streak below one disables the guard. +func ApplyLoginStreak(cls Classification, streak, required int) Classification { + if cls.Class != ClassAuthRequired || required <= 1 || streak >= required { + return cls + } + return Classification{ClassUnknown, fmt.Sprintf("login chrome seen %d/%d polls", streak, required)} +} diff --git a/src/crux/watchdog/classify_login_grace_test.go b/src/crux/watchdog/classify_login_grace_test.go index 5174f2f1e8..dcd5dc2360 100644 --- a/src/crux/watchdog/classify_login_grace_test.go +++ b/src/crux/watchdog/classify_login_grace_test.go @@ -114,3 +114,89 @@ func TestClassifyLoginPromptObservationInsideBootGrace(t *testing.T) { t.Errorf("past grace: Classify() = %v (%q), want %v", got.Class, got.Reason, ClassAuthRequired) } } + +// TestClassifyLoginChromeAfterMidLifeRelaunch pins #17320: 22h into their +// lives, three copilot agents had their CLIs relaunched and each flashed login +// chrome for ~0.5s while re-validating the token. StartedAt dated the agent, +// not the process, so the relaunch got no grace and one sighting paged. +func TestClassifyLoginChromeAfterMidLifeRelaunch(t *testing.T) { + s := classifySettings() + base := Observation{ + Backend: "copilot", + SessionExists: true, + Pane: "Please run /login to authenticate", + ShowsLoginPrompt: true, + StartedAt: fresh(22 * time.Hour), + LastChange: fresh(time.Second), + } + + cases := []struct { + name string + lastLaunchAt time.Time + want PaneClass + }{ + {"relaunch seconds ago is inside grace", fresh(3 * time.Second), ClassUnknown}, + {"relaunch past grace pages", fresh(s.BootGrace + time.Second), ClassAuthRequired}, + {"no relaunch recorded falls back to StartedAt", time.Time{}, ClassAuthRequired}, + // A stale LastLaunchAt (older than StartedAt) must never shrink the + // grace a fresh StartedAt already grants. + {"older relaunch than StartedAt does not win", fresh(48 * time.Hour), ClassAuthRequired}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + obs := base + obs.LastLaunchAt = tc.lastLaunchAt + if got := Classify(obs, testClock, s); got.Class != tc.want { + t.Errorf("Classify() = %v (%q), want %v", got.Class, got.Reason, tc.want) + } + }) + } + + t.Run("relaunch grace also covers a missing session", func(t *testing.T) { + obs := base + obs.SessionExists = false + obs.LastLaunchAt = fresh(3 * time.Second) + if got := Classify(obs, testClock, s); got.Class != ClassUnknown { + t.Errorf("Classify() = %v (%q), want %v", got.Class, got.Reason, ClassUnknown) + } + }) + t.Run("fresh StartedAt still grants grace when LastLaunchAt is older", func(t *testing.T) { + obs := base + obs.StartedAt = fresh(3 * time.Second) + obs.LastLaunchAt = fresh(time.Hour) + if got := Classify(obs, testClock, s); got.Class != ClassUnknown { + t.Errorf("Classify() = %v (%q), want %v", got.Class, got.Reason, ClassUnknown) + } + }) +} + +// TestApplyLoginStreak is the classifier-side half of the relaunch guard: a +// relaunch the manager cannot date still flashes login chrome, so the verdict +// must survive several consecutive polls before it pages. +func TestApplyLoginStreak(t *testing.T) { + auth := Classification{ClassAuthRequired, "pane shows a login/credential screen"} + cases := []struct { + name string + in Classification + streak int + required int + want PaneClass + wantReason string + }{ + {"first sighting is unknown", auth, 1, 3, ClassUnknown, "login chrome seen 1/3 polls"}, + {"second sighting is unknown", auth, 2, 3, ClassUnknown, "login chrome seen 2/3 polls"}, + {"streak at threshold pages", auth, 3, 3, ClassAuthRequired, auth.Reason}, + {"streak past threshold keeps paging", auth, 7, 3, ClassAuthRequired, auth.Reason}, + {"required of one disables the guard", auth, 1, 1, ClassAuthRequired, auth.Reason}, + {"required of zero disables the guard", auth, 1, 0, ClassAuthRequired, auth.Reason}, + {"non-auth verdicts pass through", Classification{ClassReady, "CLI ready signature present"}, 0, 3, ClassReady, "CLI ready signature present"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := ApplyLoginStreak(tc.in, tc.streak, tc.required) + if got.Class != tc.want || got.Reason != tc.wantReason { + t.Errorf("ApplyLoginStreak(%v, %d, %d) = %v (%q), want %v (%q)", tc.in.Class, tc.streak, tc.required, got.Class, got.Reason, tc.want, tc.wantReason) + } + }) + } +} diff --git a/src/crux/watchdog/login_streak.go b/src/crux/watchdog/login_streak.go new file mode 100644 index 0000000000..20e68509bb --- /dev/null +++ b/src/crux/watchdog/login_streak.go @@ -0,0 +1,20 @@ +package watchdog + +import "time" + +// classifyLocked runs the pane state machine for one agent and layers the +// reconciler's per-agent login streak on top: auth-required only stands once +// the pane has shown login chrome on Settings.LoginStreak consecutive polls. +// A relaunched CLI flashes login chrome for a fraction of a second while it +// re-validates its token, and the manager cannot date every relaunch (the CLI +// re-executing itself, for one), so one sighting must not page an operator +// (#17320). Caller holds r.mu. +func (r *Reconciler) classifyLocked(rec *agentRecord, obs Observation, now time.Time) Classification { + cls := Classify(obs, now, r.settings) + if cls.Class == ClassAuthRequired { + rec.loginStreak++ + } else { + rec.loginStreak = 0 + } + return ApplyLoginStreak(cls, rec.loginStreak, r.settings.LoginStreak) +} diff --git a/src/crux/watchdog/login_streak_test.go b/src/crux/watchdog/login_streak_test.go new file mode 100644 index 0000000000..219dbcac22 --- /dev/null +++ b/src/crux/watchdog/login_streak_test.go @@ -0,0 +1,87 @@ +package watchdog + +import ( + "context" + "strings" + "testing" +) + +// TestAuthLoginStreak pins the reconciler half of #17320: with the default +// LoginStreak, login chrome must persist across consecutive polls before +// Authenticated flips False and an operator is paged. A relaunched CLI that +// flashes login chrome on one poll and is back at its prompt on the next never +// pages; a pane genuinely stuck at login still does once the streak is met. +func TestAuthLoginStreak(t *testing.T) { + loginObs := Observation{ + Backend: "copilot", SessionExists: true, + Pane: loginPane, ShowsLoginPrompt: true, + StartedAt: launchedLongAgo, + } + readyObs := Observation{ + Backend: "copilot", SessionExists: true, + Pane: "● reviewing PR #12", HasCLIMarker: true, + StartedAt: launchedLongAgo, + } + settings := fastSettings() + settings.LoginStreak = DefaultLoginStreak + + t.Run("one-poll relaunch flash never pages", func(t *testing.T) { + fleet := newFakeFleet("a1") + alerter := newFakeAlerter() + r := newTestReconciler(t, settings, fleet, alerter, newFakeClock()) + + fleet.setObs("a1", loginObs) + r.Tick(context.Background()) + auth, _ := FindCondition(r.Conditions("a1"), ConditionAuthenticated) + if auth.Reason == "PaneShowsLogin" { + t.Fatalf("Authenticated = %+v after one sighting, want not PaneShowsLogin", auth) + } + if alerter.has(authAlertID("a1")) { + t.Fatal("one login sighting paged the operator") + } + if got := r.Conditions("a1"); !hasReadyReasonContaining(got, "login chrome seen 1/3 polls") { + t.Fatalf("Ready condition should carry the streak reason, got %+v", got) + } + + fleet.setObs("a1", readyObs) + r.Tick(context.Background()) + fleet.setObs("a1", loginObs) + r.Tick(context.Background()) + fleet.setObs("a1", loginObs) + r.Tick(context.Background()) + // Two sightings after a reset are still below the threshold. + if alerter.has(authAlertID("a1")) { + t.Fatal("streak must reset when the pane leaves login chrome") + } + }) + + t.Run("login chrome on every poll pages at the threshold", func(t *testing.T) { + fleet := newFakeFleet("a1") + alerter := newFakeAlerter() + r := newTestReconciler(t, settings, fleet, alerter, newFakeClock()) + fleet.setObs("a1", loginObs) + + for i := 1; i < DefaultLoginStreak; i++ { + r.Tick(context.Background()) + if alerter.has(authAlertID("a1")) { + t.Fatalf("paged after %d/%d sightings", i, DefaultLoginStreak) + } + } + r.Tick(context.Background()) + auth, _ := FindCondition(r.Conditions("a1"), ConditionAuthenticated) + if auth.Status != ConditionFalse || auth.Reason != "PaneShowsLogin" { + t.Fatalf("Authenticated = %+v at the streak threshold, want False/PaneShowsLogin", auth) + } + if !alerter.has(authAlertID("a1")) { + t.Fatal("a pane stuck at login must still page once the streak is met") + } + if fleet.restartCount() != 0 { + t.Fatal("auth-required must never restart") + } + }) +} + +func hasReadyReasonContaining(conds []Condition, want string) bool { + ready, ok := FindCondition(conds, ConditionReady) + return ok && strings.Contains(ready.Message, want) +} diff --git a/src/crux/watchdog/reconciler.go b/src/crux/watchdog/reconciler.go index d8650fd142..94946f54ef 100644 --- a/src/crux/watchdog/reconciler.go +++ b/src/crux/watchdog/reconciler.go @@ -225,9 +225,9 @@ type agentRecord struct { // alert fire once, not every tick. CrashLooping bool // LastClass is the previous liveness verdict, for transition logging. - LastClass PaneClass - // restartInFlight is true while a restart goroutine is running; the - // reconciler never stacks a second restart on a wedged one. + LastClass PaneClass + loginStreak int // consecutive polls showing login chrome; see classifyLocked + // restartInFlight: a restart goroutine is running; never stack another on a wedged one. restartInFlight bool // restartStartedAt dates the in-flight restart so a wedge past the hard // timeout is alerted on (the control-plane guard for failure mode 4). @@ -503,11 +503,11 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, name string, now time.T r.logger.Warn("watchdog: agent unobservable", "agent", name, "error", err) return } - cls := Classify(obs, now, r.snapshotSettings()) paused := r.fleet.IsPaused(name) r.mu.Lock() rec := r.recordLocked(name) + cls := r.classifyLocked(rec, obs, now) if cls.Class != rec.LastClass { r.logger.Info("watchdog: liveness transition", "agent", name, "from", string(rec.LastClass), "to", string(cls.Class), "reason", cls.Reason) diff --git a/src/crux/watchdog/reconciler_test.go b/src/crux/watchdog/reconciler_test.go index 42a0620105..6ce04bb243 100644 --- a/src/crux/watchdog/reconciler_test.go +++ b/src/crux/watchdog/reconciler_test.go @@ -308,6 +308,9 @@ func fastSettings() Settings { s.Mode = ModeHeal s.ProbeInterval = 0 // every Tick sweeps; interval gating has its own test s.RestartTimeout = 50 * time.Millisecond + // One sighting pages: the auth tests here assert the paging branch itself; + // the multi-poll streak guard has its own test (TestAuthLoginStreak). + s.LoginStreak = 1 return s } diff --git a/src/crux/watchdog/settings.go b/src/crux/watchdog/settings.go index df9f6a3cca..1202e3a61c 100644 --- a/src/crux/watchdog/settings.go +++ b/src/crux/watchdog/settings.go @@ -39,6 +39,12 @@ const ( // Kept numerically equal to that constant rather than importing it, // because pkg/agent imports pkg/watchdog and the reverse would cycle. DefaultBootGrace = 60 * time.Second + // DefaultLoginStreak is how many consecutive watchdog polls must see + // login chrome before the pane is classified auth-required and an + // operator is paged. Mirrors the pane poller's loginStreakRestartMin: a + // relaunched CLI flashes login chrome for well under a second while it + // re-validates its token, and a single sighting is not a logout (#17320). + DefaultLoginStreak = 3 ) // defaultBackoff is the RFC restart ladder; the last entry is the cap. @@ -130,6 +136,7 @@ type Settings struct { StuckOverlayAfter time.Duration ShellPromptAfter time.Duration BootGrace time.Duration + LoginStreak int Backoff []time.Duration CrashLoopAfter int HealthyReset time.Duration @@ -154,6 +161,7 @@ func DefaultSettings() Settings { StuckOverlayAfter: DefaultStuckOverlayAfter, ShellPromptAfter: DefaultShellPromptAfter, BootGrace: DefaultBootGrace, + LoginStreak: DefaultLoginStreak, Backoff: defaultBackoff(), CrashLoopAfter: DefaultCrashLoopAfter, HealthyReset: DefaultHealthyReset, @@ -207,6 +215,11 @@ func SettingsFrom(cfg config.WatchdogConfig) (Settings, []error) { s.StuckOverlayAfter = resolveDuration("watchdog.liveness.stuck_overlay_after", cfg.Liveness.StuckOverlayAfter, DefaultStuckOverlayAfter, &errs) s.ShellPromptAfter = resolveDuration("watchdog.liveness.shell_prompt_after", cfg.Liveness.ShellPromptAfter, DefaultShellPromptAfter, &errs) + if cfg.Liveness.LoginStreak < 0 { + errs = append(errs, fmt.Errorf("watchdog.liveness.login_streak: negative value %d, using default %d", cfg.Liveness.LoginStreak, DefaultLoginStreak)) + } else if cfg.Liveness.LoginStreak > 0 { + s.LoginStreak = cfg.Liveness.LoginStreak + } s.HealthyReset = resolveDuration("watchdog.restart.healthy_reset", cfg.Restart.HealthyReset, DefaultHealthyReset, &errs) s.NoProductionFor = resolveDuration("watchdog.readiness.no_production_for", cfg.Readiness.NoProductionFor, DefaultNoProductionFor, &errs) diff --git a/src/crux/watchdog/settings_test.go b/src/crux/watchdog/settings_test.go index 44fa32f327..3447e1cce8 100644 --- a/src/crux/watchdog/settings_test.go +++ b/src/crux/watchdog/settings_test.go @@ -23,6 +23,7 @@ func TestSettingsFromZeroValueIsRFCDefaults(t *testing.T) { s.StuckOverlayAfter != want.StuckOverlayAfter || s.ShellPromptAfter != want.ShellPromptAfter || s.CrashLoopAfter != want.CrashLoopAfter || + s.LoginStreak != want.LoginStreak || s.HealthyReset != want.HealthyReset || s.NoProductionFor != want.NoProductionFor || s.RestartTimeout != want.RestartTimeout { @@ -41,6 +42,7 @@ func TestSettingsFromExplicitValues(t *testing.T) { Liveness: config.WatchdogLivenessConfig{ StuckOverlayAfter: "3m", ShellPromptAfter: "90s", + LoginStreak: 5, }, Restart: config.WatchdogRestartConfig{ Backoff: []string{"30s", "1m"}, @@ -58,7 +60,7 @@ func TestSettingsFromExplicitValues(t *testing.T) { t.Fatal("explicit enabled:false must map to mode off, and auth_probe:false must win") } if s.ProbeInterval != time.Minute || s.StuckOverlayAfter != 3*time.Minute || - s.ShellPromptAfter != 90*time.Second || s.CrashLoopAfter != 2 || + s.ShellPromptAfter != 90*time.Second || s.CrashLoopAfter != 2 || s.LoginStreak != 5 || s.HealthyReset != 10*time.Minute || s.NoProductionFor != 2*time.Hour { t.Fatalf("explicit values not honored: %+v", s) } @@ -73,6 +75,7 @@ func TestSettingsFromInvalidValuesFallBackLoudly(t *testing.T) { Liveness: config.WatchdogLivenessConfig{ StuckOverlayAfter: "not-a-duration", ShellPromptAfter: "-5m", + LoginStreak: -1, }, Restart: config.WatchdogRestartConfig{ Backoff: []string{"1m", "banana"}, @@ -83,14 +86,16 @@ func TestSettingsFromInvalidValuesFallBackLoudly(t *testing.T) { } s, errs := SettingsFrom(cfg) // One error per invalid field: probe interval, two liveness durations, - // backoff entry, crash-loop count, healthy reset, no-production-for. - const wantErrs = 7 + // login streak, backoff entry, crash-loop count, healthy reset, + // no-production-for. + const wantErrs = 8 if len(errs) != wantErrs { t.Fatalf("want %d config errors, got %d: %v", wantErrs, len(errs), errs) } want := DefaultSettings() if s.ProbeInterval != want.ProbeInterval || s.StuckOverlayAfter != want.StuckOverlayAfter || s.ShellPromptAfter != want.ShellPromptAfter || s.CrashLoopAfter != want.CrashLoopAfter || + s.LoginStreak != want.LoginStreak || s.HealthyReset != want.HealthyReset || s.NoProductionFor != want.NoProductionFor { t.Fatalf("invalid fields must fall back to defaults: %+v", s) } diff --git a/src/docs/agent-watchdog.md b/src/docs/agent-watchdog.md index 5cccabdcd7..bd6493d35f 100644 --- a/src/docs/agent-watchdog.md +++ b/src/docs/agent-watchdog.md @@ -28,9 +28,16 @@ launched, running, unpaused agent: restarts until a human (or the resume API) intervenes. A continuous-ready window of `healthy_reset` (default 30m) clears the failure counter. 4. **Auth probing** — a pane sitting at a login screen sets - `Authenticated=False` and raises an alert but deliberately does **not** - restart: restarting into a dead credential is the 1042-restart loop the - RFC documents. Provider-level credential probes reuse the rotation + `Authenticated=False` (reason `PaneShowsLogin`) and raises an alert but + deliberately does **not** restart: restarting into a dead credential is + the 1042-restart loop the RFC documents. Two guards keep a CLI's own + startup handshake from paging anyone: login chrome is ignored inside the + boot grace of the agent's most recent CLI (re)launch — a kick restart or + memory-drain recycle hours into an agent's life boots exactly like a fresh + start — and it must persist for `liveness.login_streak` consecutive polls + (default 3; one sighting reports `Ready=Unknown` with reason + `login chrome seen 1/3 polls`) before it counts, mirroring the pane + poller's own `loginStreak` guard. Provider-level credential probes reuse the rotation package's probers (Claude OAuth usage, Codex app-server handshake, Agy headless usage, DeepSeek balance) and the owner-aware per-agent credential check (per-agent `$HOME` layouts, 0600 session files). @@ -125,6 +132,7 @@ governor: liveness: stuck_overlay_after: 10m shell_prompt_after: 5m + login_streak: 3 # consecutive polls of login chrome before auth-required restart: backoff: [1m, 2m, 4m, 8m, 16m] crash_loop_after: 5 diff --git a/src/pkg/config/testdata/agent_runtime_golden.json b/src/pkg/config/testdata/agent_runtime_golden.json index d089227a15..b5201ad0d5 100644 --- a/src/pkg/config/testdata/agent_runtime_golden.json +++ b/src/pkg/config/testdata/agent_runtime_golden.json @@ -265,7 +265,8 @@ "ProbeIntervalS": 45, "Liveness": { "StuckOverlayAfter": "10m", - "ShellPromptAfter": "3m" + "ShellPromptAfter": "3m", + "LoginStreak": 0 }, "Restart": { "Backoff": [ diff --git a/src/pkg/dashboard/static-src/js/config-governor.js b/src/pkg/dashboard/static-src/js/config-governor.js index 5fc0bed3c5..454d97282e 100644 --- a/src/pkg/dashboard/static-src/js/config-governor.js +++ b/src/pkg/dashboard/static-src/js/config-governor.js @@ -1780,7 +1780,7 @@
Observes each agent's actual terminal rather than its configured state, and can restart agents that are running but dead.
-YAML keys: governor.watchdog.enabled, governor.watchdog.mode, governor.watchdog.auth_probe, governor.watchdog.probe_interval_s, governor.watchdog.liveness.shell_prompt_after, governor.watchdog.liveness.stuck_overlay_after, governor.watchdog.readiness.no_production_for, governor.watchdog.restart.backoff[], governor.watchdog.restart.crash_loop_after, governor.watchdog.restart.healthy_reset.
YAML keys: governor.watchdog.enabled, governor.watchdog.mode, governor.watchdog.auth_probe, governor.watchdog.probe_interval_s, governor.watchdog.liveness.login_streak, governor.watchdog.liveness.shell_prompt_after, governor.watchdog.liveness.stuck_overlay_after, governor.watchdog.readiness.no_production_for, governor.watchdog.restart.backoff[], governor.watchdog.restart.crash_loop_after, governor.watchdog.restart.healthy_reset.
Observes each agent's actual terminal rather than its configured state, and can restart agents that are running but dead.
-YAML keys: governor.watchdog.enabled, governor.watchdog.mode, governor.watchdog.auth_probe, governor.watchdog.probe_interval_s, governor.watchdog.liveness.shell_prompt_after, governor.watchdog.liveness.stuck_overlay_after, governor.watchdog.readiness.no_production_for, governor.watchdog.restart.backoff[], governor.watchdog.restart.crash_loop_after, governor.watchdog.restart.healthy_reset.
YAML keys: governor.watchdog.enabled, governor.watchdog.mode, governor.watchdog.auth_probe, governor.watchdog.probe_interval_s, governor.watchdog.liveness.login_streak, governor.watchdog.liveness.shell_prompt_after, governor.watchdog.liveness.stuck_overlay_after, governor.watchdog.readiness.no_production_for, governor.watchdog.restart.backoff[], governor.watchdog.restart.crash_loop_after, governor.watchdog.restart.healthy_reset.