From fb9c69a28bd1d383c49f38a9087c90e50d37ff29 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 17 Nov 2026 12:41:58 -0500 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=90=9B=20claude:=20keep=20the=20rotat?= =?UTF-8?q?ed=20shared-login=20grant=20until=20it=20is=20durably=20written?= =?UTF-8?q?;=20one=20writer=20for=20the=20shared=20login?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The provider kills the old refresh token the instant it answers, so a failed write of the new credential lost the login for the whole fleet. The new grant is now saved to a private .credentials.pending file first, held in memory and retried with backoff without calling the token endpoint again, adopted at boot after a crash, and a dashboard banner and log line say so when it cannot be saved. invalid_grant is no longer final while a newer grant exists. The refresher, the agents' bridge and the session adopt-back write the shared login through one only-if-strictly-newer function under one lock. Fixes #18775 Signed-off-by: Andy Anderson --- ...-18775-claude-shared-refresh-lost-write.md | 1 + src/cmd/hive/claude_shared_login_alert.go | 37 ++ src/cmd/hive/main_eval_cycle.go | 1 + src/crux/agent/claude_credential_bridge.go | 22 +- src/crux/agent/claude_shared_grant.go | 346 ++++++++++++++++++ src/crux/agent/claude_shared_grant_test.go | 337 +++++++++++++++++ src/crux/agent/claude_shared_refresh.go | 61 ++- src/crux/agent/credentials_watchdog.go | 6 + src/docs/security-model.md | 10 +- src/docs/troubleshooting.md | 52 +++ 10 files changed, 814 insertions(+), 59 deletions(-) create mode 100644 changelog.d/fixed-18775-claude-shared-refresh-lost-write.md create mode 100644 src/cmd/hive/claude_shared_login_alert.go create mode 100644 src/crux/agent/claude_shared_grant.go create mode 100644 src/crux/agent/claude_shared_grant_test.go diff --git a/changelog.d/fixed-18775-claude-shared-refresh-lost-write.md b/changelog.d/fixed-18775-claude-shared-refresh-lost-write.md new file mode 100644 index 0000000000..0625e15094 --- /dev/null +++ b/changelog.d/fixed-18775-claude-shared-refresh-lost-write.md @@ -0,0 +1 @@ +- The hive's own refresh of the shared Claude login can no longer lose the rotated grant when the write fails: the new grant is saved to a private `.credentials.pending` file before the shared file is written, held and retried with backoff without calling the token endpoint again, adopted at boot after a crash, and a dashboard banner and log line say so when it cannot be saved; an `invalid_grant` is no longer final while a newer grant from an agent or session home exists, and the refresher, the agents' bridge and the session adopt-back now write the shared login through one "only if strictly newer" function (#18775). diff --git a/src/cmd/hive/claude_shared_login_alert.go b/src/cmd/hive/claude_shared_login_alert.go new file mode 100644 index 0000000000..9375ee2c56 --- /dev/null +++ b/src/cmd/hive/claude_shared_login_alert.go @@ -0,0 +1,37 @@ +package main + +// claudeSharedLoginUnsavedAlertID is the dashboard system-alert id for "the +// hive refreshed the shared Claude login but could not write the new grant" +// (hivecommons/hive#18775). Raised and cleared level-triggered, once per +// evaluation cycle, from the agent manager's own account of the held grant. +const claudeSharedLoginUnsavedAlertID = "claude-shared-login-unsaved" + +// claudeSharedLoginUnsavedSource is implemented by the agent manager. +type claudeSharedLoginUnsavedSource interface { + ClaudeSharedLoginUnsavedAlert() string +} + +// claudeSharedLoginUnsavedSink is the subset of *dashboard.Server used. +type claudeSharedLoginUnsavedSink interface { + AddSystemAlert(id, severity, message string) + ClearSystemAlert(id string) +} + +// reconcileClaudeSharedLoginUnsavedAlert raises the banner while a refreshed +// grant is held unwritten and clears it once it is saved. A runtime that does +// not hold grants (any other crux.Runtime) leaves it cleared. +func reconcileClaudeSharedLoginUnsavedAlert(sink claudeSharedLoginUnsavedSink, rt any) { + if sink == nil { + return + } + src, ok := rt.(claudeSharedLoginUnsavedSource) + if !ok { + sink.ClearSystemAlert(claudeSharedLoginUnsavedAlertID) + return + } + if msg := src.ClaudeSharedLoginUnsavedAlert(); msg != "" { + sink.AddSystemAlert(claudeSharedLoginUnsavedAlertID, "error", msg) + return + } + sink.ClearSystemAlert(claudeSharedLoginUnsavedAlertID) +} diff --git a/src/cmd/hive/main_eval_cycle.go b/src/cmd/hive/main_eval_cycle.go index 2ba797be4b..7e5b031f5c 100644 --- a/src/cmd/hive/main_eval_cycle.go +++ b/src/cmd/hive/main_eval_cycle.go @@ -1355,6 +1355,7 @@ func runEvalCycle( // (#18732) are waiting for an operator sign-in, not a restart: keep the // dashboard banner level with the runtime's BackendAuth marks. dashSrv.ReconcileClaudeLoginRevokedAlert(agentMgr.AllStatuses()) + reconcileClaudeSharedLoginUnsavedAlert(dashSrv, agentMgr) // Epoch captured before reading agent/governor state so a mutation that // lands mid-build (restart-count/budget reset) drops this snapshot instead diff --git a/src/crux/agent/claude_credential_bridge.go b/src/crux/agent/claude_credential_bridge.go index 7483c959d9..d79a91d2a1 100644 --- a/src/crux/agent/claude_credential_bridge.go +++ b/src/crux/agent/claude_credential_bridge.go @@ -475,27 +475,13 @@ func (m *Manager) relinkUnreadableAfterSignIn(agentName, link, target string) (* // metadata by descriptor, and a refusal (sharedfile.ErrUnsafeLocation) when // the shared directory or file is not what it should be. func promoteClaudeCredential(data []byte, target string, now time.Time) (string, error) { - canonical, _, err := canonicalClaudeCredential(data, now) + _, tok, err := canonicalClaudeCredential(data, now) if err != nil { return "", err } - // Refuse before the backup copy, not only at the write: the copy lands in - // the same directory, and the directory is part of what is being checked. - info, err := sharedfile.CheckLocation(target) - if err != nil { - return "", err - } - staleShared := "" - if info != nil { - staleShared, err = preserveClaudeCredential(target, now) - if err != nil { - return "", fmt.Errorf("preserve shared credential: %w", err) - } - } - if err := sharedfile.Replace(target, canonical, claudeSharedCredentialMode, sharedfile.Options{PreserveOwner: true}); err != nil { - return staleShared, err - } - return staleShared, nil + // The one writer of the shared login: re-checks, under the caller's + // claudeBridgeMu, that tok is strictly newer (claude_shared_grant.go). + return replaceSharedClaudeLoginIfNewer(tok, target, now, true) } // preserveClaudeCredential copies path to a unique stale name beside it, diff --git a/src/crux/agent/claude_shared_grant.go b/src/crux/agent/claude_shared_grant.go new file mode 100644 index 0000000000..6748ef25f1 --- /dev/null +++ b/src/crux/agent/claude_shared_grant.go @@ -0,0 +1,346 @@ +package agent + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/hivecommons/hive/crux/claude" + "github.com/hivecommons/hive/crux/sharedfile" +) + +// Durable hand-off of a freshly rotated Claude grant, and the one function +// every writer of the shared login goes through (hivecommons/hive#18775, +// follow-up to #18731). +// +// The lost-write window. The provider rotates the refresh token on a +// successful refresh: the instant the token endpoint answers, the refresh +// token the hive sent is dead and the new one exists only in the response. If +// the write of the new credential then failed, and the grant were dropped, the +// next attempt would present the dead token, get invalid_grant, and the whole +// fleet would be signed out until a person signs in again. So the new grant is +// +// 1. written to a second, private file beside the shared one +// (.credentials.pending, 0600, fsynced) BEFORE the shared file is touched; +// 2. held in memory until the shared file has it, and written with backoff +// (claudeSharedWriteBackoffBase doubling to claudeSharedWriteBackoffMax, +// with no overall limit while the process lives); +// 3. protected from the refresher itself: while a grant is held the token +// endpoint is never called again, whatever the clock says. +// +// A restart finds the pending file and adopts it if it is newer than the +// shared file (settlePendingSharedClaudeGrant). If the grant cannot be written +// anywhere the hive says so loudly (log, audit, and the dashboard banner from +// ClaudeSharedLoginUnsavedAlert) while the access token it replaces still has +// life left. +// +// Closed or narrowed? Inside a running process the window is closed: nothing +// drops the grant and nothing presents the old token again. Across a crash it +// is narrowed, not closed: it is the time between the token endpoint's answer +// reaching this process and the pending file's fsync completing (milliseconds +// when the disk works), plus the case where BOTH files fail to write and the +// process dies before either recovers. A hive cannot do better than that +// without the provider's cooperation, because the old token is dead from the +// moment of the answer. + +const ( + // claudeSharedPendingName is the pending grant's file name, beside the + // shared credential. It is deliberately not ".credentials.json", so the + // permissions watcher that widens that name to group-readable never + // touches it. + claudeSharedPendingName = ".credentials.pending" + // claudeSharedPendingMode: the hive's own uid only. The pending grant holds + // a live refresh token and agents have no business reading it. + claudeSharedPendingMode = 0o600 + + // claudeSharedWriteBackoffBase is the wait after the first failed write of a + // held grant; it doubles per failure up to claudeSharedWriteBackoffMax. + claudeSharedWriteBackoffBase = 5 * time.Second + claudeSharedWriteBackoffMax = 5 * time.Minute + + // claudeSharedReconcileTrigger labels the bridge and session sweeps run + // before a grant is declared rejected. + claudeSharedReconcileTrigger = "refresh-reconcile" +) + +// errClaudeSharedNotNewer: the shared login already holds a usable grant at +// least as new as the candidate, so the candidate is not written. +var errClaudeSharedNotNewer = errors.New("the shared Claude login already holds a grant at least as new") + +// claudeSharedWrite and claudeSharedWritePending are the write seams; tests +// substitute failures to prove the hold-and-retry behaviour. +var ( + claudeSharedWrite = replaceSharedClaudeLoginIfNewer + claudeSharedWritePending = writeSharedClaudePending +) + +// replaceSharedClaudeLoginIfNewer is THE writer of the shared Claude login: the +// central refresher, the agents' credential bridge (promote-on-race) and the +// session adopt-back all end here. The caller holds claudeBridgeMu, so the +// read of the current login and the write that replaces it are one step. +// +// cand replaces the shared login only if it is complete and usable +// (claudeBridgePromotable) and the shared login is not usable or expires +// strictly earlier than cand; otherwise nothing is written +// (errClaudeBridgeNotPromotable, errClaudeSharedNotNewer). Only the typed +// token block is written, atomically, fsynced, at claudeSharedCredentialMode +// and owned like the file it replaces. With preserveOld the replaced file is +// kept beside it first and its path returned (the refresher passes false: its +// refresh token is dead the moment the server rotated it). +func replaceSharedClaudeLoginIfNewer(cand *claude.OAuthTokens, target string, now time.Time, preserveOld bool) (string, error) { + if !claudeBridgePromotable(cand, now) { + return "", errClaudeBridgeNotPromotable + } + if cur := claude.LoadTokens(target); cur != nil && claude.TokensUsable(cur, now) && cand.ExpiresAt <= cur.ExpiresAt { + return "", errClaudeSharedNotNewer + } + data, err := json.Marshal(claude.Credentials{ClaudeAIOAuth: cand}) + if err != nil { + return "", errClaudeBridgeNotPromotable + } + // Refuse before any backup copy, not only at the write: the copy lands in + // the same directory, and the directory is part of what is being checked. + info, err := sharedfile.CheckLocation(target) + if err != nil { + return "", err + } + staleShared := "" + if preserveOld && info != nil { + if staleShared, err = preserveClaudeCredential(target, now); err != nil { + return "", fmt.Errorf("preserve shared credential: %w", err) + } + } + if err := sharedfile.Replace(target, data, claudeSharedCredentialMode, sharedfile.Options{PreserveOwner: true}); err != nil { + return staleShared, err + } + return staleShared, nil +} + +// sharedClaudePendingPath is the pending grant file beside the shared login. +func sharedClaudePendingPath(target string) string { + return filepath.Join(filepath.Dir(target), claudeSharedPendingName) +} + +// writeSharedClaudePending durably stores tok as the pending grant: temp file, +// fsync, rename, directory fsync, mode 0600. +func writeSharedClaudePending(tok *claude.OAuthTokens, target string) error { + data, err := json.Marshal(claude.Credentials{ClaudeAIOAuth: tok}) + if err != nil { + return err + } + return sharedfile.Replace(sharedClaudePendingPath(target), data, claudeSharedPendingMode, sharedfile.Options{}) +} + +// readSharedClaudePending returns the pending grant, nil when absent or not a +// credential. +func readSharedClaudePending(target string) *claude.OAuthTokens { + data, err := sharedfile.ReadNoFollow(sharedClaudePendingPath(target), claudeBridgeMaxCredentialBytes) + if err != nil { + return nil + } + return claude.ParseTokens(data) +} + +// removeSharedClaudePending deletes the pending file; absent is fine. +func removeSharedClaudePending(target string) { + _ = os.Remove(sharedClaudePendingPath(target)) +} + +// holdSharedClaudeGrant records fresh as the grant that must reach the shared +// file before anything else happens. The pending file is written first; if +// that fails the grant is still held in memory and the failure is logged. +func (m *Manager) holdSharedClaudeGrant(fresh *claude.OAuthTokens, target string, lifetime time.Duration) { + st := &m.claudeSharedRefresh + st.mu.Lock() + st.pending, st.pendingLifetime, st.pendingOnDisk = fresh, lifetime, false + st.writeFailures, st.writeNotBefore, st.writeErr = 0, time.Time{}, "" + st.mu.Unlock() + m.ensureSharedClaudePendingOnDisk(fresh, target) +} + +// ensureSharedClaudePendingOnDisk writes the pending file if it is not there. +func (m *Manager) ensureSharedClaudePendingOnDisk(p *claude.OAuthTokens, target string) { + st := &m.claudeSharedRefresh + st.mu.Lock() + done := st.pendingOnDisk + st.mu.Unlock() + if done { + return + } + if err := claudeSharedWritePending(p, target); err != nil { + m.logger.Error("claude shared login: could not save the new refresh grant to its pending file; it is held in memory only until the shared file is written", + "pending", sharedClaudePendingPath(target), "error", err) + return + } + st.mu.Lock() + st.pendingOnDisk = true + st.mu.Unlock() +} + +// settlePendingSharedClaudeGrant writes a held grant to the shared file, or +// adopts one a previous process left in the pending file. handled is true when +// a grant is held (even while it waits out a backoff): the caller must not +// call the token endpoint. +func (m *Manager) settlePendingSharedClaudeGrant(now time.Time) (claudeSharedRefreshOutcome, bool) { + target := sharedClaudeBridgeTarget() + st := &m.claudeSharedRefresh + st.mu.Lock() + if !st.pendingChecked { + st.pendingChecked = true + if st.pending == nil { + st.mu.Unlock() + p := readSharedClaudePending(target) + st.mu.Lock() + if p != nil && st.pending == nil { + st.pending, st.pendingOnDisk = p, true + m.logger.Warn("claude shared login: found a pending refresh grant from an earlier run; writing it to the shared login if it is newer", "pending", sharedClaudePendingPath(target)) + } + } + } + held, wait := st.pending != nil, now.Before(st.writeNotBefore) + st.mu.Unlock() + if !held { + return claudeSharedRefreshSkipped, false + } + if wait { + return claudeSharedRefreshSkipped, true + } + return m.flushPendingSharedClaudeGrant(now), true +} + +// flushPendingSharedClaudeGrant makes one attempt to write the held grant. +func (m *Manager) flushPendingSharedClaudeGrant(now time.Time) claudeSharedRefreshOutcome { + target := sharedClaudeBridgeTarget() + st := &m.claudeSharedRefresh + claudeBridgeMu.Lock() + st.mu.Lock() + p, lifetime := st.pending, st.pendingLifetime + st.mu.Unlock() + if p == nil { + claudeBridgeMu.Unlock() + return claudeSharedRefreshSkipped + } + m.ensureSharedClaudePendingOnDisk(p, target) + _, err := claudeSharedWrite(p, target, now, false) + claudeBridgeMu.Unlock() + + switch { + case err == nil: + m.clearSharedClaudePending(target) + m.mu.Lock() + m.claudeAuthToken = p.AccessToken + m.mu.Unlock() + st.mu.Lock() + if lifetime > 0 { + st.lifetime = lifetime + } + st.failedExpiresAt, st.notBefore, st.rejectedExpiresAt = 0, time.Time{}, 0 + st.mu.Unlock() + expires := time.UnixMilli(p.ExpiresAt).UTC().Format(time.RFC3339) + m.logger.Info("claude shared login: refreshed the shared Claude credential ahead of expiry; agents read the new token through their credential bridges and do not refresh it themselves", + "shared", target, "expires_at", expires) + m.audit(AuditClaudeCredentialRefreshed, "", auditFields("outcome", "success", "trigger", "timer", "expires_at", expires)) + return claudeSharedRefreshWritten + case errors.Is(err, errClaudeSharedNotNewer): + m.clearSharedClaudePending(target) + m.logger.Info("claude shared login: a fresher credential reached the shared file while the hive was refreshing it; keeping the fresher one", "shared", target) + return claudeSharedRefreshSuperseded + case errors.Is(err, errClaudeBridgeNotPromotable): + m.clearSharedClaudePending(target) + m.logger.Error("claude shared login: the held refresh grant is no longer usable and was dropped; sign in again if the fleet reports a revoked login", "shared", target) + return claudeSharedRefreshFailed + } + m.recordSharedClaudeWriteFailure(err, now, target) + return claudeSharedRefreshFailed +} + +// clearSharedClaudePending forgets the held grant: the file first, then the +// memory, so a new hold can never be deleted by an old clear. +func (m *Manager) clearSharedClaudePending(target string) { + removeSharedClaudePending(target) + st := &m.claudeSharedRefresh + st.mu.Lock() + st.pending, st.pendingOnDisk, st.pendingLifetime = nil, false, 0 + st.writeFailures, st.writeNotBefore, st.writeErr = 0, time.Time{}, "" + st.mu.Unlock() +} + +// recordSharedClaudeWriteFailure arms the backoff and says, loudly and +// specifically, that the new grant is not yet saved. The error carries paths +// and errno text only, never token material. +func (m *Manager) recordSharedClaudeWriteFailure(err error, now time.Time, target string) { + st := &m.claudeSharedRefresh + st.mu.Lock() + st.writeFailures++ + n := st.writeFailures + delay := claudeSharedWriteBackoffBase + for i := 1; i < n && delay < claudeSharedWriteBackoffMax; i++ { + delay *= 2 + } + if delay > claudeSharedWriteBackoffMax { + delay = claudeSharedWriteBackoffMax + } + st.writeNotBefore, st.writeErr = now.Add(delay), err.Error() + onDisk := st.pendingOnDisk + st.mu.Unlock() + + m.logger.Error("claude shared login: the hive redeemed the shared Claude grant but could NOT save the new one; it is held and retried, and the token endpoint will not be called again until it is saved. Fix the credentials directory (disk space, ownership), or sign in again from the dashboard before the current access token expires", + "shared", target, "error", err, "attempt", n, "retry_in", delay.String(), "pending_file_saved", onDisk) + if n == 1 { + m.audit(AuditClaudeCredentialRefreshed, "", auditFields( + "outcome", "failed", "trigger", "timer", "reason", "new grant could not be saved", + "pending_file_saved", onDisk, "recovery", defaultCredentialRecovery)) + } +} + +// ClaudeSharedLoginUnsavedAlert is the dashboard banner text while a refreshed +// grant is held but not yet written to the shared login, empty otherwise. The +// host polls it (the dashboard cannot be imported here) and raises or clears +// its system alert. Never carries token material. +func (m *Manager) ClaudeSharedLoginUnsavedAlert() string { + st := &m.claudeSharedRefresh + st.mu.Lock() + failures, errText, onDisk, held := st.writeFailures, st.writeErr, st.pendingOnDisk, st.pending != nil + st.mu.Unlock() + if !held || failures == 0 { + return "" + } + deadline := "unknown" + if cur := claude.LoadTokens(sharedClaudeBridgeTarget()); cur != nil && cur.ExpiresAt > 0 { + deadline = time.UnixMilli(cur.ExpiresAt).UTC().Format(time.RFC3339) + } + backup := "NOT saved to its backup file either, so a restart now would lose it" + if onDisk { + backup = "saved to a backup file (.credentials.pending) and adopted on restart" + } + return fmt.Sprintf("The hive refreshed the shared Claude login but cannot write the new one (%s, %d attempts). The new grant is held in memory and %s. "+ + "The login the agents read expires at %s: fix the credentials directory (disk space, ownership) before then, or sign in to Claude from the dashboard.", + errText, failures, backup, deadline) +} + +// reconcileBeforeRejectingSharedClaude runs when the token endpoint refused +// the shared refresh token. A refusal is final only if no newer grant exists: +// the bridge and the session adopt-back are asked to publish any newer +// credential from an agent or session home first, and a shared login that now +// holds a different refresh token is a newer grant, so the refused one is not +// marked rejected. Called with no lock held. +func (m *Manager) reconcileBeforeRejectingSharedClaude(rejected *claude.OAuthTokens, target string, now time.Time) bool { + st := &m.claudeSharedRefresh + st.mu.Lock() + held := st.pending != nil + st.mu.Unlock() + if held { + return true + } + m.RepairClaudeCredentialBridges(claudeSharedReconcileTrigger) + m.syncUserSessionClaudeLogins(claudeSharedReconcileTrigger) + cur := claude.LoadTokens(target) + if cur == nil || !claudeBridgePromotable(cur, now) || cur.RefreshToken == rejected.RefreshToken { + return false + } + m.logger.Info("claude shared login: the refresh grant was refused, but a different, newer grant has reached the shared file; not marking the login rejected", "shared", target) + return true +} diff --git a/src/crux/agent/claude_shared_grant_test.go b/src/crux/agent/claude_shared_grant_test.go new file mode 100644 index 0000000000..86098aaa6a --- /dev/null +++ b/src/crux/agent/claude_shared_grant_test.go @@ -0,0 +1,337 @@ +package agent + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/claude" +) + +// All credentials here are synthetic and the token endpoint is +// fakeRefreshServer; nothing real is read, written or contacted. + +// grantClock replaces the refresher's clock with one the test advances. +func grantClock(t *testing.T) *time.Time { + t.Helper() + clock := time.Now() + prev := claudeSharedRefreshNow + claudeSharedRefreshNow = func() time.Time { return clock } + t.Cleanup(func() { claudeSharedRefreshNow = prev }) + return &clock +} + +// grantWriteSeams swaps the shared-file and pending-file writers for ones +// that fail while *failMain / *failPending are true. +func grantWriteSeams(t *testing.T, failMain, failPending *bool) { + t.Helper() + realMain, realPending := claudeSharedWrite, claudeSharedWritePending + claudeSharedWrite = func(c *claude.OAuthTokens, target string, now time.Time, preserve bool) (string, error) { + if *failMain { + return "", errors.New("write credentials: no space left on device") + } + return realMain(c, target, now, preserve) + } + claudeSharedWritePending = func(c *claude.OAuthTokens, target string) error { + if *failPending { + return errors.New("write pending: no space left on device") + } + return realPending(c, target) + } + t.Cleanup(func() { claudeSharedWrite, claudeSharedWritePending = realMain, realPending }) +} + +func grantLogs(m *Manager) *bytes.Buffer { + var buf bytes.Buffer + m.logger = slog.New(slog.NewTextHandler(&buf, nil)) + return &buf +} + +func assertNoSentinel(t *testing.T, what, s string) { + t.Helper() + if strings.Contains(s, bridgeTestTokenSentinel) { + t.Errorf("token material reached %s", what) + } +} + +func assertNoTokenInAudit(t *testing.T, sink *fakeAuditSink) { + t.Helper() + sink.mu.Lock() + defer sink.mu.Unlock() + for _, e := range sink.entries { + assertNoSentinel(t, "the audit log", fmt.Sprint(e.Fields)) + } +} + +// The new grant is written, the endpoint is called exactly once, and a write +// that fails once does not cost the login. +func TestSharedClaudeGrant_WriteFailsOnceThenSucceeds(t *testing.T) { + m, sink, _, target := sharedRefreshFixture(t, 1, time.Now().Add(time.Minute).UnixMilli()) + srv := &fakeRefreshServer{live: readTokensAt(t, target).RefreshToken, lifetime: 8 * time.Hour} + withFakeRefresh(t, srv) + logs := grantLogs(m) + clock := grantClock(t) + failMain, failPending := true, false + grantWriteSeams(t, &failMain, &failPending) + + if got := m.refreshSharedClaudeCredential(); got != claudeSharedRefreshFailed { + t.Fatalf("first pass = %v, want failed", got) + } + if tok := readTokensAt(t, target); !strings.HasPrefix(tok.AccessToken, "acc-shared-") { + t.Fatal("shared file changed although the write failed") + } + pend := readSharedClaudePending(target) + if pend == nil || !strings.HasPrefix(pend.AccessToken, "acc-rotated-") { + t.Fatal("the new grant was not saved to the pending file before the write") + } + if info, err := os.Stat(sharedClaudePendingPath(target)); err != nil || info.Mode().Perm() != claudeSharedPendingMode { + t.Errorf("pending file mode = %v err=%v, want 0600", info.Mode(), err) + } + if m.ClaudeSharedLoginUnsavedAlert() == "" { + t.Error("no alert while the new grant is unsaved") + } + + // Inside the backoff nothing is attempted, and never the endpoint. + if got := m.refreshSharedClaudeCredential(); got != claudeSharedRefreshSkipped { + t.Errorf("pass inside the backoff = %v, want skipped", got) + } + failMain = false + *clock = clock.Add(claudeSharedWriteBackoffBase + time.Second) + if got := m.refreshSharedClaudeCredential(); got != claudeSharedRefreshWritten { + t.Fatalf("retry = %v, want written", got) + } + if srv.calls.Load() != 1 { + t.Errorf("token endpoint called %d times, want exactly once", srv.calls.Load()) + } + if tok := readTokensAt(t, target); !strings.HasPrefix(tok.AccessToken, "acc-rotated-") { + t.Error("the new grant never reached the shared file") + } + if info, err := os.Stat(target); err != nil || info.Mode().Perm() != claudeSharedCredentialMode { + t.Errorf("shared file mode = %v err=%v", info.Mode(), err) + } + if readSharedClaudePending(target) != nil { + t.Error("pending file left behind after the write succeeded") + } + if m.ClaudeSharedLoginUnsavedAlert() != "" { + t.Error("alert still raised after the write succeeded") + } + assertNoSentinel(t, "the log", logs.String()) + assertNoTokenInAudit(t, sink) +} + +// The process "restarts" after the refresh and before the write: the pending +// grant is adopted by the next process without another refresh. +func TestSharedClaudeGrant_PendingAdoptedAfterRestart(t *testing.T) { + m1, _, _, target := sharedRefreshFixture(t, 1, time.Now().Add(time.Minute).UnixMilli()) + srv := &fakeRefreshServer{live: readTokensAt(t, target).RefreshToken, lifetime: 8 * time.Hour} + withFakeRefresh(t, srv) + failMain, failPending := true, false + grantWriteSeams(t, &failMain, &failPending) + grantClock(t) + m1.refreshSharedClaudeCredential() // refresh succeeds, write fails, "crash" + + failMain = false + m2, sink := testManagerWithSink(t) + logs := grantLogs(m2) + if got := m2.refreshSharedClaudeCredential(); got != claudeSharedRefreshWritten { + t.Fatalf("pass after restart = %v, want written", got) + } + if srv.calls.Load() != 1 { + t.Errorf("token endpoint called %d times across the restart, want once", srv.calls.Load()) + } + if tok := readTokensAt(t, target); !strings.HasPrefix(tok.AccessToken, "acc-rotated-") { + t.Error("the pending grant was not adopted") + } + if readSharedClaudePending(target) != nil { + t.Error("pending file left behind after adoption") + } + assertNoSentinel(t, "the log", logs.String()) + assertNoTokenInAudit(t, sink) +} + +// A pending grant that is not newer than the shared login is dropped, never +// written over it. +func TestSharedClaudeGrant_StalePendingIsDroppedNotWritten(t *testing.T) { + m, _, _, target := sharedRefreshFixture(t, 1, time.Now().Add(8*time.Hour).UnixMilli()) + older := &claude.OAuthTokens{AccessToken: "acc-old-" + bridgeTestTokenSentinel, RefreshToken: "ref-old-" + bridgeTestTokenSentinel, + ExpiresAt: time.Now().Add(time.Hour).UnixMilli()} + if err := writeSharedClaudePending(older, target); err != nil { + t.Fatal(err) + } + srv := &fakeRefreshServer{live: "unused", lifetime: 8 * time.Hour} + withFakeRefresh(t, srv) + m.settlePendingSharedClaudeGrant(time.Now()) + if tok := readTokensAt(t, target); !strings.HasPrefix(tok.AccessToken, "acc-shared-") { + t.Error("an older pending grant overwrote the shared login") + } + if readSharedClaudePending(target) != nil { + t.Error("stale pending file not removed") + } + if srv.calls.Load() != 0 { + t.Error("token endpoint called") + } +} + +// The write keeps failing: the endpoint is never called again, the alert is +// raised, and nothing secret is logged, audited or put in the banner. +func TestSharedClaudeGrant_WriteKeepsFailing_NoSecondRefreshAndAlert(t *testing.T) { + m, sink, _, target := sharedRefreshFixture(t, 1, time.Now().Add(time.Minute).UnixMilli()) + srv := &fakeRefreshServer{live: readTokensAt(t, target).RefreshToken, lifetime: 8 * time.Hour} + withFakeRefresh(t, srv) + logs := grantLogs(m) + clock := grantClock(t) + failMain, failPending := true, false + grantWriteSeams(t, &failMain, &failPending) + + for i := 0; i < 8; i++ { + m.refreshSharedClaudeCredential() + *clock = clock.Add(claudeSharedWriteBackoffMax + time.Second) // past any backoff, and past the old credential's expiry + } + if srv.calls.Load() != 1 { + t.Fatalf("token endpoint called %d times while a newer grant was unwritten, want once", srv.calls.Load()) + } + alert := m.ClaudeSharedLoginUnsavedAlert() + if alert == "" || !strings.Contains(alert, "no space left on device") || !strings.Contains(alert, ".credentials.pending") { + t.Errorf("alert = %q", alert) + } + if !strings.Contains(logs.String(), "could NOT save the new one") { + t.Error("no loud log line") + } + rec := sink.find(AuditClaudeCredentialRefreshed) + if rec == nil || rec.Fields["reason"] != "new grant could not be saved" { + t.Errorf("audit entry = %+v", rec) + } + assertNoSentinel(t, "the log", logs.String()) + assertNoSentinel(t, "the alert", alert) + assertNoTokenInAudit(t, sink) + + // Both locations failing is said plainly. + failPending = true + m2, _ := testManagerWithSink(t) + m2.holdSharedClaudeGrant(&claude.OAuthTokens{AccessToken: "a", RefreshToken: "r", ExpiresAt: time.Now().Add(time.Hour).UnixMilli()}, target, time.Hour) + m2.recordSharedClaudeWriteFailure(errors.New("boom"), time.Now(), target) + if a := m2.ClaudeSharedLoginUnsavedAlert(); !strings.Contains(a, "NOT saved to its backup file") { + t.Errorf("alert with both writes failing = %q", a) + } +} + +// invalid_grant while a newer grant has reached an agent's home: reconciled, +// not marked rejected. +func TestSharedClaudeGrant_InvalidGrantWithNewerGrantIsReconciled(t *testing.T) { + m, sink, links, target := sharedRefreshFixture(t, 1, time.Now().Add(time.Minute).UnixMilli()) + srv := &fakeRefreshServer{live: "something-else", lifetime: 8 * time.Hour} // refuses the shared refresh token + withFakeRefresh(t, srv) + logs := grantLogs(m) + later := time.Now().Add(9 * time.Hour).UnixMilli() + simulateClaudeRefresh(t, links[0], bridgeTestCreds(t, "agent", later)) + + if got := m.refreshSharedClaudeCredential(); got != claudeSharedRefreshSuperseded { + t.Fatalf("outcome = %v, want superseded (reconciled)", got) + } + if tok := readTokensAt(t, target); !strings.HasPrefix(tok.AccessToken, "acc-agent-") { + t.Error("the newer agent grant was not promoted before judging the refusal") + } + m.claudeSharedRefresh.mu.Lock() + rejected := m.claudeSharedRefresh.rejectedExpiresAt + m.claudeSharedRefresh.mu.Unlock() + if rejected != 0 { + t.Error("credential marked rejected although a newer grant exists") + } + if rec := sink.find(AuditClaudeCredentialRefreshed); rec != nil && rec.Fields["reason"] == "refresh grant rejected" { + t.Error("a rejection was audited") + } + assertNoSentinel(t, "the log", logs.String()) + assertNoTokenInAudit(t, sink) +} + +// The central refresher, the bridge's promote-on-race and the session +// adopt-back all end in replaceSharedClaudeLoginIfNewer under claudeBridgeMu: +// whatever the arrival order, the newest grant wins. +func TestSharedClaudeGrant_ThreeWritersNewestAlwaysWins(t *testing.T) { + shared := withSharedAgentHome(t) + target := sharedClaudeBridgeTarget() + if err := os.MkdirAll(shared+"/.claude", 0o755); err != nil { + t.Fatal(err) + } + now := time.Now() + grant := func(tag string, d time.Duration) *claude.OAuthTokens { + return &claude.OAuthTokens{AccessToken: "acc-" + tag + "-" + bridgeTestTokenSentinel, + RefreshToken: "ref-" + tag + "-" + bridgeTestTokenSentinel, ExpiresAt: now.Add(d).UnixMilli()} + } + grants := []*claude.OAuthTokens{grant("w1", 1*time.Hour), grant("w2", 2*time.Hour), grant("w3", 3*time.Hour)} + for iter := 0; iter < 12; iter++ { + if err := os.WriteFile(target, bridgeTestCreds(t, "base", now.Add(30*time.Minute).UnixMilli()), 0o660); err != nil { + t.Fatal(err) + } + m, _ := testManagerWithSink(t) + g := func(i int) *claude.OAuthTokens { return grants[(i+iter)%3] } + m.claudeSharedRefresh.pending = g(0) + var wg sync.WaitGroup + wg.Add(3) + go func() { defer wg.Done(); m.flushPendingSharedClaudeGrant(now) }() // the refresher + go func() { // the agents' bridge promote-on-race + defer wg.Done() + data := mustMarshalCreds(t, g(1)) + claudeBridgeMu.Lock() + defer claudeBridgeMu.Unlock() + _, _ = promoteClaudeCredential(data, target, now) + }() + go func() { // the session adopt-back + defer wg.Done() + data := mustMarshalCreds(t, g(2)) + claudeBridgeMu.Lock() + defer claudeBridgeMu.Unlock() + _, _ = promoteClaudeCredential(data, target, now) + }() + wg.Wait() + if got := readTokensAt(t, target); got.ExpiresAt != grants[2].ExpiresAt { + t.Fatalf("iteration %d: shared login expires %d, want the newest %d", iter, got.ExpiresAt, grants[2].ExpiresAt) + } + } +} + +func TestReplaceSharedClaudeLoginIfNewer_Rule(t *testing.T) { + shared := withSharedAgentHome(t) + target := sharedClaudeBridgeTarget() + if err := os.MkdirAll(shared+"/.claude", 0o755); err != nil { + t.Fatal(err) + } + now := time.Now() + cur := now.Add(2 * time.Hour) + if err := os.WriteFile(target, bridgeTestCreds(t, "cur", cur.UnixMilli()), 0o660); err != nil { + t.Fatal(err) + } + mk := func(exp time.Time) *claude.OAuthTokens { + return &claude.OAuthTokens{AccessToken: "a", RefreshToken: "r", ExpiresAt: exp.UnixMilli()} + } + for _, c := range []*claude.OAuthTokens{mk(cur.Add(-time.Minute)), mk(cur)} { + if _, err := replaceSharedClaudeLoginIfNewer(c, target, now, false); !errors.Is(err, errClaudeSharedNotNewer) { + t.Errorf("older or equal grant: err = %v, want errClaudeSharedNotNewer", err) + } + } + if _, err := replaceSharedClaudeLoginIfNewer(&claude.OAuthTokens{AccessToken: "a", ExpiresAt: 1}, target, now, false); !errors.Is(err, errClaudeBridgeNotPromotable) { + t.Errorf("incomplete grant: err = %v", err) + } + if _, err := replaceSharedClaudeLoginIfNewer(mk(cur.Add(time.Minute)), target, now, false); err != nil { + t.Fatalf("newer grant: %v", err) + } + if got := readTokensAt(t, target); got.ExpiresAt != cur.Add(time.Minute).UnixMilli() { + t.Error("newer grant not written") + } +} + +func mustMarshalCreds(t *testing.T, tok *claude.OAuthTokens) []byte { + t.Helper() + data, err := json.Marshal(claude.Credentials{ClaudeAIOAuth: tok}) + if err != nil { + t.Fatal(err) + } + return data +} diff --git a/src/crux/agent/claude_shared_refresh.go b/src/crux/agent/claude_shared_refresh.go index 0476eac818..c8ce3d6ac3 100644 --- a/src/crux/agent/claude_shared_refresh.go +++ b/src/crux/agent/claude_shared_refresh.go @@ -124,6 +124,15 @@ type claudeSharedRefreshState struct { // tick (or a concurrent caller) does not present the same refresh token // a second time, which is precisely the reuse that revokes the grant. inFlight bool + + // A refreshed grant not yet in the shared file (claude_shared_grant.go). + pending *claude.OAuthTokens + pendingLifetime time.Duration + pendingOnDisk bool + pendingChecked bool + writeFailures int + writeNotBefore time.Time + writeErr string } // claudeSharedRefreshEnabled reports whether the central refresh is on @@ -208,6 +217,11 @@ func (m *Manager) refreshSharedClaudeCredential() claudeSharedRefreshOutcome { } target := sharedClaudeBridgeTarget() now := claudeSharedRefreshNow() + // A grant already redeemed but not yet written comes first: the token + // endpoint is never called while one is held (claude_shared_grant.go). + if out, held := m.settlePendingSharedClaudeGrant(now); held { + return out + } tok := claude.LoadTokens(target) st := &m.claudeSharedRefresh @@ -229,6 +243,9 @@ func (m *Manager) refreshSharedClaudeCredential() claudeSharedRefreshOutcome { fresh, err := claudeRefreshTokens(tok) if err != nil { + if errors.Is(err, claude.ErrRefreshGrantRejected) && m.reconcileBeforeRejectingSharedClaude(tok, target, now) { + return claudeSharedRefreshSuperseded + } m.recordSharedClaudeRefreshFailure(tok.ExpiresAt, now, err) return claudeSharedRefreshFailed } @@ -237,45 +254,11 @@ func (m *Manager) refreshSharedClaudeCredential() claudeSharedRefreshOutcome { return claudeSharedRefreshFailed } - // Write under the bridge lock so a promotion racing with this pass and - // this write cannot interleave; keep the later-expiring credential. - claudeBridgeMu.Lock() - current := claude.LoadTokens(target) - if current != nil && current.ExpiresAt > fresh.ExpiresAt { - claudeBridgeMu.Unlock() - m.logger.Info("claude shared login: a fresher credential reached the shared file while the hive was refreshing it; keeping the fresher one", - "shared", target, "expires_at", time.UnixMilli(current.ExpiresAt).UTC().Format(time.RFC3339)) - return claudeSharedRefreshSuperseded - } - werr := claude.WriteCredentials(fresh, target) - claudeBridgeMu.Unlock() - if werr != nil { - m.recordSharedClaudeRefreshFailure(tok.ExpiresAt, now, werr) - return claudeSharedRefreshFailed - } - - // Agents launched from now on that cannot read the file (manager_env.go) - // should get this token, not the one cached at boot. m.mu is never taken - // while claudeBridgeMu is held; it is released above. - m.mu.Lock() - m.claudeAuthToken = fresh.AccessToken - m.mu.Unlock() - - st.mu.Lock() - st.lifetime = time.Duration(fresh.ExpiresAt-now.UnixMilli()) * time.Millisecond - st.failedExpiresAt, st.notBefore, st.rejectedExpiresAt = 0, time.Time{}, 0 - st.mu.Unlock() - - expires := time.UnixMilli(fresh.ExpiresAt).UTC().Format(time.RFC3339) - m.logger.Info("claude shared login: refreshed the shared Claude credential ahead of expiry; agents read the new token through their credential bridges and do not refresh it themselves", - "shared", target, "expires_at", expires, "lead", lead.String(), - "previous_expires_at", time.UnixMilli(tok.ExpiresAt).UTC().Format(time.RFC3339)) - m.audit(AuditClaudeCredentialRefreshed, "", auditFields( - "outcome", "success", - "trigger", "timer", - "expires_at", expires, - )) - return claudeSharedRefreshWritten + // The old refresh token is dead from here on: keep the new grant on disk + // and in memory until the shared file has it, then write it under the + // bridge lock with the same "strictly newer wins" rule as every writer. + m.holdSharedClaudeGrant(fresh, target, time.Duration(fresh.ExpiresAt-now.UnixMilli())*time.Millisecond) + return m.flushPendingSharedClaudeGrant(now) } // recordSharedClaudeRefreshFailure logs and audits one failed pass and arms diff --git a/src/crux/agent/credentials_watchdog.go b/src/crux/agent/credentials_watchdog.go index e4ebad400c..bff49dbb8a 100644 --- a/src/crux/agent/credentials_watchdog.go +++ b/src/crux/agent/credentials_watchdog.go @@ -250,6 +250,12 @@ func (m *Manager) StartCredentialWatchdog(ctx context.Context) { // then runs on every bridge poll. It is not gated on an agent using // Claude: sessions use the shared login either way. m.syncUserSessionClaudeLogins(sessionLoginTriggerBoot) + // A refreshed grant an earlier run saved but never got into the shared + // file is adopted now, before anything can present the dead token + // (claude_shared_grant.go). + if !m.claudeOperatorTokenMode() { + m.settlePendingSharedClaudeGrant(claudeSharedRefreshNow()) + } for { select { case <-ctx.Done(): diff --git a/src/docs/security-model.md b/src/docs/security-model.md index 24dfb02fe8..200e90d026 100644 --- a/src/docs/security-model.md +++ b/src/docs/security-model.md @@ -537,8 +537,14 @@ importance: therefore gets the Claude login **without its refresh token**, and only the login's own token block, nothing else stored in that file. It can use the fleet's current access token and cannot rotate or carry away the long-lived - grant. Refreshing stays where it was: with the agents' credential bridge and - the dashboard sign-in, the only writers of the shared file. The hive keeps + grant. Who refreshes it: the hive itself, once, ahead of expiry + ([#18731](https://github.com/hivecommons/hive/issues/18731); it redeems the + shared refresh token and holds the new grant on disk and in memory until the + shared file has it, `HIVE_CLAUDE_SHARED_REFRESH=off` disables it), the + agents' credential bridge when a CLI refreshed first anyway, and the + dashboard sign-in. A session never refreshes. All writers of the shared file + go through one function under one lock that accepts only a usable credential + expiring strictly later than the current one. The hive keeps the copies current (it replaces them, atomically and as the session's uid, when the shared login changes), so a session depends on the fleet's login being refreshed by an agent or a sign-in; a session cannot do that itself. diff --git a/src/docs/troubleshooting.md b/src/docs/troubleshooting.md index 41369ffbad..99095191dd 100644 --- a/src/docs/troubleshooting.md +++ b/src/docs/troubleshooting.md @@ -701,6 +701,58 @@ expiry`, and the audit log records `claude_credential_refreshed` with bridge promotes its copy exactly as before (`kept=agent`). File contents are never logged. `HIVE_CLAUDE_SHARED_REFRESH=off` turns the central refresh off. +Kill switch: with `HIVE_CLAUDE_SHARED_REFRESH=off` the hive never calls the +token endpoint for the shared login. Each CLI then refreshes for itself, and +the only protection is the after-the-fact repair by the agents' credential +bridge and the session adopt-back, which cannot stop N CLIs from racing one +refresh token. Use it only if the hive's own refresh misbehaves; a pending +grant left by an earlier run is still adopted at boot with it off. + +The new refresh token must not be lost. The provider kills the old refresh +token the moment it answers, so the hive saves the new grant to +`.credentials.pending` (mode 0600, fsynced, beside the shared file) before it +writes the shared file, holds it in memory, and retries the write with backoff +(5 s doubling to 5 min, with no overall limit) without calling the token +endpoint again. After a restart a pending grant newer than the shared file is +adopted at boot; an older one is deleted. If the write keeps failing the hive +logs `claude shared login: the hive redeemed the shared Claude grant but could +NOT save the new one`, records `claude_credential_refreshed` with +`reason=new grant could not be saved`, and raises the dashboard banner +"The hive refreshed the shared Claude login but cannot write the new one", +which names the time the login the agents read expires. Fix the disk space or +ownership of `/data/home/.claude`, or sign in from the dashboard, before then. +This closes the window while the process lives; across a crash it narrows it to +the milliseconds between the provider's answer and the pending file's fsync, +or to the case where both files fail and the process dies before either +recovers. A refused grant is also not final while a newer one exists: before +marking the login rejected the hive publishes any newer credential from an +agent or session home and compares the shared file again. + +Everything that writes the shared login (this refresher, the agents' bridge +promote-on-race, the session adopt-back) goes through one function under one +lock, which writes only a usable credential that expires strictly later than +the current one. Only the typed token block is ever written; any other content +in the file is dropped, as the dashboard sign-in does, on purpose. + +Live check (a real CLI is needed; the unit tests cannot cover it). The hive +assumes a running Claude CLI re-reads the rewritten file through its +credential symlink instead of refreshing with a refresh token it cached; if +it does not, its later refresh presents a dead token and trips the provider's +reuse detection. The hive does not restart agents on a refresh. To confirm, on +a hive with at least two Claude agents: + +1. Note each agent's pane and set `HIVE_CLAUDE_SHARED_REFRESH_LEAD` to a value + just under the current remaining lifetime so the next minute's pass refreshes + (or wait for the natural pass). +2. Watch for `claude shared login: refreshed the shared Claude credential` + and the `claude_credential_refreshed` audit entry with `outcome=success`. +3. Leave the agents running past the OLD access token's expiry and send each a + task. They must keep working with no `OAuth token revoked`, no login prompt, + and no `claude credential bridge ... re-bridged` entry with `kept=agent` + (that entry means a CLI refreshed on its own). +4. If any agent shows the prompt, or a `kept=agent` entry appears, a running + CLI is not picking the file up: set `HIVE_CLAUDE_SHARED_REFRESH=off` and report it. + If the shared grant is already dead, the hive's own refresh is rejected too: the log says `claude shared login: the authorization server rejected the shared refresh grant` From 734ef58afb6d17d017672d89c14dfc1854a1edea Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 17 Nov 2026 13:10:26 -0500 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=8C=B1=20retrigger=20CI:=20PR=20Issue?= =?UTF-8?q?=20check-run=20did=20not=20complete?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson