Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-watchdog-login-flash-relaunch.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Watchdog no longer pages `PaneShowsLogin` for the sub-second login flash a relaunched Copilot CLI paints while re-validating its token: boot grace is now dated from the most recent CLI (re)launch, and login chrome must persist for `governor.watchdog.liveness.login_streak` consecutive polls (default 3) before `auth-required` is raised.
7 changes: 4 additions & 3 deletions src/crux/agent/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -427,9 +427,10 @@ type AgentProcess struct {
// this as "Waiting on CI" instead of "Working" so a stalled-on-CI agent
// no longer looks indistinguishable from one making progress.
AwaitingCI bool
TransientNudges int // total transient-API-error nudges sent (surfaced to the dashboard)
launchGen int // increments per launch; stale deliverStartupKick goroutines check it and drop
lastInferKickMarks int // no-action watchdog: tool-marker count in pane+scrollback just after kick delivery
TransientNudges int // total transient-API-error nudges sent (surfaced to the dashboard)
launchGen int // increments per launch; stale deliverStartupKick goroutines check it and drop
lastCLILaunchAt time.Time // when the CLI was last typed into the pane; dates watchdog boot grace on relaunch (#17320)
lastInferKickMarks int // no-action watchdog: tool-marker count in pane+scrollback just after kick delivery
ProviderErrorClass string
ProviderErrorLine string
ProviderErrorBackoffUntil time.Time
Expand Down
3 changes: 1 addition & 2 deletions src/crux/agent/manager_launch.go
Original file line number Diff line number Diff line change
Expand Up @@ -313,8 +313,7 @@ func (m *Manager) launchInTmux(ctx context.Context, agent *AgentProcess) error {
agent.LastError = ""
agent.copilotModelFallbackRequested = ""
agent.lastLaunchFailureBanner = ""
agent.StartedAt = &now
agent.launchGen++
m.markCLILaunchedLocked(agent, now)
m.logger.Info("audit: agent started",
"name", agent.Name,
"backend", backend,
Expand Down
16 changes: 15 additions & 1 deletion src/crux/agent/watchdog_fleet.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,14 +144,17 @@ func (f WatchdogFleet) Observe(name string) (watchdog.Observation, error) {

// StartedAt dates the current launch so the reconciler can suppress dead
// verdicts during boot. Copied by value: the field is a pointer the
// manager mutates on relaunch.
// manager mutates on relaunch. LastLaunchAt is the most recent CLI spawn
// (markCLILaunchedLocked), which a restored or resumed session can leave
// StartedAt lagging behind.
var startedAt time.Time
providerClass := ""
providerLine := ""
f.M.mu.RLock()
if agent.StartedAt != nil {
startedAt = *agent.StartedAt
}
lastLaunchAt := agent.lastCLILaunchAt
if f.M.providerErrorBackoffRemainingLocked(agent, time.Now()) > 0 {
providerClass = agent.ProviderErrorClass
providerLine = agent.ProviderErrorLine
Expand All @@ -166,6 +169,7 @@ func (f WatchdogFleet) Observe(name string) (watchdog.Observation, error) {
ShowsLoginPrompt: needsLogin || paneShowsLoginPrompt(strings.Split(pane, "\n")),
LastChange: lastChange,
StartedAt: startedAt,
LastLaunchAt: lastLaunchAt,
AuthAvailable: authAvailable,
AuthKnown: authKnown,
CredentialProven: credentialProven,
Expand Down Expand Up @@ -439,3 +443,13 @@ func newestMtimeBounded(root string, maxDepth int) (time.Time, bool) {
func (m *Manager) WatchdogSource() watchdog.Source {
return WatchdogFleet{M: m}
}

// markCLILaunchedLocked records a completed CLI launch into the pane: the
// agent's StartedAt, the launch generation stale startup-kick goroutines
// check, and the relaunch timestamp the watchdog dates its boot grace from
// (#17320). Caller holds m.mu.
func (m *Manager) markCLILaunchedLocked(agent *AgentProcess, now time.Time) {
agent.StartedAt = &now
agent.launchGen++
agent.lastCLILaunchAt = now
}
6 changes: 6 additions & 0 deletions src/crux/config/watchdog.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ type WatchdogLivenessConfig struct {
// window exists because a launching CLI legitimately passes through both
// states. "" = 5m.
ShellPromptAfter string `yaml:"shell_prompt_after,omitempty"`
// LoginStreak is how many consecutive watchdog polls must see login
// chrome before the pane is classified auth-required and an operator is
// paged; fewer sightings report Unknown. A relaunched CLI flashes login
// chrome while re-validating its token, so one sighting is not a logout.
// 0 = 3; 1 disables the guard.
LoginStreak int `yaml:"login_streak,omitempty"`
}

// WatchdogRestartConfig holds the CrashLoopBackOff-analog knobs.
Expand Down
47 changes: 43 additions & 4 deletions src/crux/watchdog/classify.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package watchdog

import (
"fmt"
"strings"
"time"
)
Expand Down Expand Up @@ -67,6 +68,13 @@ type Observation struct {
// agent spawns a second concurrent CLI, the race cliBootGraceSeconds was
// added for on the manager side.
StartedAt time.Time
// LastLaunchAt is when the CLI process was most recently (re)launched
// into the pane; zero when the manager has not recorded one. StartedAt
// dates the agent, not the process: a kick restart, memory-drain recycle
// or watchdog restart hours into an agent's life spawns a fresh CLI that
// repeats the whole startup handshake (login chrome included), so boot
// grace is measured from the later of the two.
LastLaunchAt time.Time
// AuthAvailable/AuthKnown carry the manager's owner-aware per-agent
// credential-file verdict (AgentAuthState, #4619/#4641): AuthKnown=false
// means the probe could not determine an answer and the reconciler must
Expand Down Expand Up @@ -208,9 +216,12 @@ func Classify(obs Observation, now time.Time, s Settings) Classification {
// Boot grace outranks every dead verdict. A launching agent legitimately
// has no tmux session, no pane, and no CLI marker for the first seconds of
// its life; without this, the watchdog would restart it underneath itself
// and spawn a second concurrent CLI. An agent with no StartedAt cannot be
// dated, so it gets no grace rather than an unbounded one.
booting := !obs.StartedAt.IsZero() && now.Sub(obs.StartedAt) < s.BootGrace
// and spawn a second concurrent CLI. An agent with no launch timestamp
// cannot be dated, so it gets no grace rather than an unbounded one. The
// grace is dated from the most recent CLI launch, not the agent's first:
// a relaunched CLI boots exactly like a new one (#17320).
launchedAt := obs.lastLaunch()
booting := !launchedAt.IsZero() && now.Sub(launchedAt) < s.BootGrace

if !obs.SessionExists {
if booting {
Expand All @@ -234,11 +245,16 @@ func Classify(obs Observation, now time.Time, s Settings) Classification {
// branch. Observed on a hosted spoke: rolling the pod paged quality,
// scanner and ci-maintainer 3 minutes in, one event each, never
// repeated -- all three were authenticated and working the whole time.
// Observed again 22h into the same agents' lives when their CLIs were
// relaunched mid-life (#17320): the grace was dated from the agent's
// first start, so the relaunch flash got none. Hence lastLaunch above,
// and the reconciler's ApplyLoginStreak on top for the relaunches the
// manager cannot date (the CLI re-executing itself, for one).
//
// Unknown rather than Ready: a login screen is not health, and this
// says only that it is too early to conclude anything. An agent that
// genuinely needs a human still shows the same chrome once grace
// elapses and pages on the very next tick.
// elapses and pages once the streak is met.
if booting {
return Classification{ClassUnknown, "pane shows login chrome inside boot grace"}
}
Expand Down Expand Up @@ -290,3 +306,26 @@ func Classify(obs Observation, now time.Time, s Settings) Classification {
}
return Classification{ClassUnknown, "pane matched no known signature"}
}

// lastLaunch is the later of the agent's start and its most recent CLI
// relaunch β€” the instant boot grace is measured from.
func (o Observation) lastLaunch() time.Time {
if o.LastLaunchAt.After(o.StartedAt) {
return o.LastLaunchAt
}
return o.StartedAt
}

// ApplyLoginStreak is the watchdog's counterpart to the pane poller's
// loginStreak guard: an auth-required verdict only stands once login chrome
// has been seen on `required` consecutive polls, `streak` being the count
// including this one. Below that it is downgraded to unknown β€” the honest
// verdict for a CLI that may simply be mid-handshake after a relaunch the
// manager could not date (#17320). Any other class passes through untouched;
// a required streak below one disables the guard.
func ApplyLoginStreak(cls Classification, streak, required int) Classification {
if cls.Class != ClassAuthRequired || required <= 1 || streak >= required {
return cls
}
return Classification{ClassUnknown, fmt.Sprintf("login chrome seen %d/%d polls", streak, required)}
}
86 changes: 86 additions & 0 deletions src/crux/watchdog/classify_login_grace_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,3 +114,89 @@ func TestClassifyLoginPromptObservationInsideBootGrace(t *testing.T) {
t.Errorf("past grace: Classify() = %v (%q), want %v", got.Class, got.Reason, ClassAuthRequired)
}
}

// TestClassifyLoginChromeAfterMidLifeRelaunch pins #17320: 22h into their
// lives, three copilot agents had their CLIs relaunched and each flashed login
// chrome for ~0.5s while re-validating the token. StartedAt dated the agent,
// not the process, so the relaunch got no grace and one sighting paged.
func TestClassifyLoginChromeAfterMidLifeRelaunch(t *testing.T) {
s := classifySettings()
base := Observation{
Backend: "copilot",
SessionExists: true,
Pane: "Please run /login to authenticate",
ShowsLoginPrompt: true,
StartedAt: fresh(22 * time.Hour),
LastChange: fresh(time.Second),
}

cases := []struct {
name string
lastLaunchAt time.Time
want PaneClass
}{
{"relaunch seconds ago is inside grace", fresh(3 * time.Second), ClassUnknown},
{"relaunch past grace pages", fresh(s.BootGrace + time.Second), ClassAuthRequired},
{"no relaunch recorded falls back to StartedAt", time.Time{}, ClassAuthRequired},
// A stale LastLaunchAt (older than StartedAt) must never shrink the
// grace a fresh StartedAt already grants.
{"older relaunch than StartedAt does not win", fresh(48 * time.Hour), ClassAuthRequired},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
obs := base
obs.LastLaunchAt = tc.lastLaunchAt
if got := Classify(obs, testClock, s); got.Class != tc.want {
t.Errorf("Classify() = %v (%q), want %v", got.Class, got.Reason, tc.want)
}
})
}

t.Run("relaunch grace also covers a missing session", func(t *testing.T) {
obs := base
obs.SessionExists = false
obs.LastLaunchAt = fresh(3 * time.Second)
if got := Classify(obs, testClock, s); got.Class != ClassUnknown {
t.Errorf("Classify() = %v (%q), want %v", got.Class, got.Reason, ClassUnknown)
}
})
t.Run("fresh StartedAt still grants grace when LastLaunchAt is older", func(t *testing.T) {
obs := base
obs.StartedAt = fresh(3 * time.Second)
obs.LastLaunchAt = fresh(time.Hour)
if got := Classify(obs, testClock, s); got.Class != ClassUnknown {
t.Errorf("Classify() = %v (%q), want %v", got.Class, got.Reason, ClassUnknown)
}
})
}

// TestApplyLoginStreak is the classifier-side half of the relaunch guard: a
// relaunch the manager cannot date still flashes login chrome, so the verdict
// must survive several consecutive polls before it pages.
func TestApplyLoginStreak(t *testing.T) {
auth := Classification{ClassAuthRequired, "pane shows a login/credential screen"}
cases := []struct {
name string
in Classification
streak int
required int
want PaneClass
wantReason string
}{
{"first sighting is unknown", auth, 1, 3, ClassUnknown, "login chrome seen 1/3 polls"},
{"second sighting is unknown", auth, 2, 3, ClassUnknown, "login chrome seen 2/3 polls"},
{"streak at threshold pages", auth, 3, 3, ClassAuthRequired, auth.Reason},
{"streak past threshold keeps paging", auth, 7, 3, ClassAuthRequired, auth.Reason},
{"required of one disables the guard", auth, 1, 1, ClassAuthRequired, auth.Reason},
{"required of zero disables the guard", auth, 1, 0, ClassAuthRequired, auth.Reason},
{"non-auth verdicts pass through", Classification{ClassReady, "CLI ready signature present"}, 0, 3, ClassReady, "CLI ready signature present"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := ApplyLoginStreak(tc.in, tc.streak, tc.required)
if got.Class != tc.want || got.Reason != tc.wantReason {
t.Errorf("ApplyLoginStreak(%v, %d, %d) = %v (%q), want %v (%q)", tc.in.Class, tc.streak, tc.required, got.Class, got.Reason, tc.want, tc.wantReason)
}
})
}
}
20 changes: 20 additions & 0 deletions src/crux/watchdog/login_streak.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package watchdog

import "time"

// classifyLocked runs the pane state machine for one agent and layers the
// reconciler's per-agent login streak on top: auth-required only stands once
// the pane has shown login chrome on Settings.LoginStreak consecutive polls.
// A relaunched CLI flashes login chrome for a fraction of a second while it
// re-validates its token, and the manager cannot date every relaunch (the CLI
// re-executing itself, for one), so one sighting must not page an operator
// (#17320). Caller holds r.mu.
func (r *Reconciler) classifyLocked(rec *agentRecord, obs Observation, now time.Time) Classification {
cls := Classify(obs, now, r.settings)
if cls.Class == ClassAuthRequired {
rec.loginStreak++
} else {
rec.loginStreak = 0
}
return ApplyLoginStreak(cls, rec.loginStreak, r.settings.LoginStreak)
}
87 changes: 87 additions & 0 deletions src/crux/watchdog/login_streak_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
package watchdog

import (
"context"
"strings"
"testing"
)

// TestAuthLoginStreak pins the reconciler half of #17320: with the default
// LoginStreak, login chrome must persist across consecutive polls before
// Authenticated flips False and an operator is paged. A relaunched CLI that
// flashes login chrome on one poll and is back at its prompt on the next never
// pages; a pane genuinely stuck at login still does once the streak is met.
func TestAuthLoginStreak(t *testing.T) {
loginObs := Observation{
Backend: "copilot", SessionExists: true,
Pane: loginPane, ShowsLoginPrompt: true,
StartedAt: launchedLongAgo,
}
readyObs := Observation{
Backend: "copilot", SessionExists: true,
Pane: "● reviewing PR #12", HasCLIMarker: true,
StartedAt: launchedLongAgo,
}
settings := fastSettings()
settings.LoginStreak = DefaultLoginStreak

t.Run("one-poll relaunch flash never pages", func(t *testing.T) {
fleet := newFakeFleet("a1")
alerter := newFakeAlerter()
r := newTestReconciler(t, settings, fleet, alerter, newFakeClock())

fleet.setObs("a1", loginObs)
r.Tick(context.Background())
auth, _ := FindCondition(r.Conditions("a1"), ConditionAuthenticated)
if auth.Reason == "PaneShowsLogin" {
t.Fatalf("Authenticated = %+v after one sighting, want not PaneShowsLogin", auth)
}
if alerter.has(authAlertID("a1")) {
t.Fatal("one login sighting paged the operator")
}
if got := r.Conditions("a1"); !hasReadyReasonContaining(got, "login chrome seen 1/3 polls") {
t.Fatalf("Ready condition should carry the streak reason, got %+v", got)
}

fleet.setObs("a1", readyObs)
r.Tick(context.Background())
fleet.setObs("a1", loginObs)
r.Tick(context.Background())
fleet.setObs("a1", loginObs)
r.Tick(context.Background())
// Two sightings after a reset are still below the threshold.
if alerter.has(authAlertID("a1")) {
t.Fatal("streak must reset when the pane leaves login chrome")
}
})

t.Run("login chrome on every poll pages at the threshold", func(t *testing.T) {
fleet := newFakeFleet("a1")
alerter := newFakeAlerter()
r := newTestReconciler(t, settings, fleet, alerter, newFakeClock())
fleet.setObs("a1", loginObs)

for i := 1; i < DefaultLoginStreak; i++ {
r.Tick(context.Background())
if alerter.has(authAlertID("a1")) {
t.Fatalf("paged after %d/%d sightings", i, DefaultLoginStreak)
}
}
r.Tick(context.Background())
auth, _ := FindCondition(r.Conditions("a1"), ConditionAuthenticated)
if auth.Status != ConditionFalse || auth.Reason != "PaneShowsLogin" {
t.Fatalf("Authenticated = %+v at the streak threshold, want False/PaneShowsLogin", auth)
}
if !alerter.has(authAlertID("a1")) {
t.Fatal("a pane stuck at login must still page once the streak is met")
}
if fleet.restartCount() != 0 {
t.Fatal("auth-required must never restart")
}
})
}

func hasReadyReasonContaining(conds []Condition, want string) bool {
ready, ok := FindCondition(conds, ConditionReady)
return ok && strings.Contains(ready.Message, want)
}
8 changes: 4 additions & 4 deletions src/crux/watchdog/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -225,9 +225,9 @@ type agentRecord struct {
// alert fire once, not every tick.
CrashLooping bool
// LastClass is the previous liveness verdict, for transition logging.
LastClass PaneClass
// restartInFlight is true while a restart goroutine is running; the
// reconciler never stacks a second restart on a wedged one.
LastClass PaneClass
loginStreak int // consecutive polls showing login chrome; see classifyLocked
// restartInFlight: a restart goroutine is running; never stack another on a wedged one.
restartInFlight bool
// restartStartedAt dates the in-flight restart so a wedge past the hard
// timeout is alerted on (the control-plane guard for failure mode 4).
Expand Down Expand Up @@ -503,11 +503,11 @@ func (r *Reconciler) reconcileAgent(ctx context.Context, name string, now time.T
r.logger.Warn("watchdog: agent unobservable", "agent", name, "error", err)
return
}
cls := Classify(obs, now, r.snapshotSettings())
paused := r.fleet.IsPaused(name)

r.mu.Lock()
rec := r.recordLocked(name)
cls := r.classifyLocked(rec, obs, now)
if cls.Class != rec.LastClass {
r.logger.Info("watchdog: liveness transition",
"agent", name, "from", string(rec.LastClass), "to", string(cls.Class), "reason", cls.Reason)
Expand Down
3 changes: 3 additions & 0 deletions src/crux/watchdog/reconciler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,9 @@ func fastSettings() Settings {
s.Mode = ModeHeal
s.ProbeInterval = 0 // every Tick sweeps; interval gating has its own test
s.RestartTimeout = 50 * time.Millisecond
// One sighting pages: the auth tests here assert the paging branch itself;
// the multi-poll streak guard has its own test (TestAuthLoginStreak).
s.LoginStreak = 1
return s
}

Expand Down
Loading
Loading