From 1903f712c273880664be6882197e1b8bddd6a8a7 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 12:21:40 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Relink=20a=20Copilot=20config.js?= =?UTF-8?q?on=20the=20CLI's=20atomic=20save=20replaced?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Copilot CLI saves config.json by renaming a temp file over it, which replaces the per-agent symlink to the shared login with a private file. Agent launch and restart now move such a copy aside (newest backup kept) and relink it; the Copilot credential probe evaluates the file the agent actually reads, and the pane poller relinks a diverged copy before the token-triggered restart. Stale config.json.tmp.* files are swept. Fixes #18443 Signed-off-by: Andy Anderson --- .../fixed-18443-copilot-config-relink.md | 1 + src/crux/agent/authprobe.go | 9 + src/crux/agent/copilot_config_bridge.go | 306 +++++++++++ src/crux/agent/copilot_config_bridge_test.go | 489 ++++++++++++++++++ src/crux/agent/interactive_home.go | 8 + src/crux/agent/manager_poll.go | 8 + src/docs/env-vars.md | 1 + src/docs/troubleshooting.md | 54 +- 8 files changed, 875 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixed-18443-copilot-config-relink.md create mode 100644 src/crux/agent/copilot_config_bridge.go create mode 100644 src/crux/agent/copilot_config_bridge_test.go diff --git a/changelog.d/fixed-18443-copilot-config-relink.md b/changelog.d/fixed-18443-copilot-config-relink.md new file mode 100644 index 0000000000..d03a35ecc4 --- /dev/null +++ b/changelog.d/fixed-18443-copilot-config-relink.md @@ -0,0 +1 @@ +- **A Copilot agent no longer stays logged out after its `~/.copilot/config.json` stops following the shared login** ([#18443](https://github.com/hivecommons/hive/issues/18443)). The Copilot CLI saves its config by renaming a temp file over the per-agent symlink, which left the agent on a private copy that never saw a new dashboard login, and restarts never repaired it. Every agent launch and restart now moves such a private copy aside to `config.json.bak-` (keeping only the newest backup) and relinks it to the shared file; the credential check behind the token-triggered restart now evaluates the file the agent actually reads, so a diverged copy is relinked instead of restarted until the hive gives up. Launch also sweeps leftover `config.json.tmp.*` files older than `HIVE_COPILOT_CONFIG_TMP_MAX_AGE` (default `1h`). A private copy that holds the only login is left in place. diff --git a/src/crux/agent/authprobe.go b/src/crux/agent/authprobe.go index ddde4a7a91..33d58ed357 100644 --- a/src/crux/agent/authprobe.go +++ b/src/crux/agent/authprobe.go @@ -345,6 +345,15 @@ func (m *Manager) credentialFileProves(agentName string, uid int, backend string } } case "copilot": + // Evaluate the file the agent's CLI actually reads (#18443). When its + // ~/.copilot/config.json is a private copy that no longer follows the + // shared login (the CLI's atomic save replaced the symlink), neither + // the shared file nor the in-memory token proves anything about this + // agent: report no proof, so the poller repairs the link instead of + // restarting a CLI that will read the same broken copy again. + if link, target, ok := copilotConfigBridgePaths(agentName, uid, backend); ok && copilotConfigDiverged(link, target) { + return false + } if m != nil { m.mu.RLock() tok := m.copilotAuthToken diff --git a/src/crux/agent/copilot_config_bridge.go b/src/crux/agent/copilot_config_bridge.go new file mode 100644 index 0000000000..195258a80c --- /dev/null +++ b/src/crux/agent/copilot_config_bridge.go @@ -0,0 +1,306 @@ +package agent + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/crux/sharedfile" +) + +// Copilot config bridge: keep each per-agent ~/.copilot/config.json linked to +// the fleet-shared login (hivecommons/hive#18443). +// +// Each per-UID agent has a REAL ~/.copilot (its own session-state) with one +// symlink in it, ~/.copilot/config.json -> /data/home/.copilot/config.json +// (perAgentCLIHomes). The Copilot CLI saves its config by writing a temp file +// and renaming it over config.json, which REPLACES the symlink with a private +// regular file. From then on the agent never sees a new login written to the +// shared file (dashboard device flow, token sync), and bridgeHomeEntry never +// repairs it because it refuses to clobber a regular file. On the Bluefin +// hosted hive one agent sat at the login prompt with a private copy that held +// no login while every other agent ran on private copies of an older token. +// +// The repair mirrors rebridgeClaudeCredential, simplified for Copilot: the +// private file is moved aside to config.json.bak- (only the +// newest backup is kept) and the symlink is restored. It runs at every agent +// launch and restart (setupPerAgentCLIHome) and when the pane poller sees a +// Copilot agent stuck at a login prompt with a diverged private copy. It never +// reads token material into a log line: only paths and outcomes are logged. +// +// One guard: when the private copy holds a token and the shared file holds +// none, the private copy may be the only login the fleet has, so it is left in +// place (logged once). A later dashboard login or token sync fills the shared +// file and the next launch relinks. + +// Repair triggers, recorded in the log. +const ( + copilotBridgeTriggerLaunch = "launch" + copilotBridgeTriggerLoginPrompt = "login-prompt" +) + +// copilotConfigBackupInfix names a private config moved aside by the repair. +const copilotConfigBackupInfix = ".bak-" + +// copilotConfigBackupNameAttempts bounds the search for a free backup name. +const copilotConfigBackupNameAttempts = 100 + +// copilotConfigMaxBytes caps a config file the divergence probe reads. A real +// Copilot config.json is a few KiB. +const copilotConfigMaxBytes = 256 << 10 + +// copilotConfigTmpInfix is the suffix the Copilot CLI's atomic save appends to +// its temp file (config.json.tmp.). A CLI killed between the write and +// the rename leaves it behind. +const copilotConfigTmpInfix = ".tmp." + +// copilotConfigTmpMaxAgeEnv overrides how old a leftover config.json.tmp.* +// must be before the launch-time sweep removes it. Go duration syntax. +const copilotConfigTmpMaxAgeEnv = "HIVE_COPILOT_CONFIG_TMP_MAX_AGE" + +// defaultCopilotConfigTmpMaxAge is old enough that no live save (milliseconds) +// can still own the file. +const defaultCopilotConfigTmpMaxAge = time.Hour + +// copilotConfigTmpSweepCap bounds one sweep, like claudeOrphanTmpSweepCap. +const copilotConfigTmpSweepCap = 200 + +// copilotBridgeNow stamps backup names and ages tmp files; a seam for tests. +var copilotBridgeNow = time.Now + +// copilotBridgeMu serializes repairs: the launch path and the poller can reach +// the same agent's link. Never held while taking m.mu (the launch path already +// holds m.mu when it provisions the home). +var copilotBridgeMu sync.Mutex + +// copilotBridgeWarned holds, per link, the last failure message logged, so a +// repair that keeps failing on every poll logs once per distinct cause. +var copilotBridgeWarned sync.Map + +// errCopilotBridgeLinkIsTarget: the agent's config path IS the shared file (a +// legacy whole-directory ~/.copilot symlink). Moving it aside would log the +// whole fleet out. +var errCopilotBridgeLinkIsTarget = errors.New("copilot config link path resolves to the shared config it would link to") + +// errCopilotBridgeSharedEmpty: the private copy holds a token and the shared +// file holds none, so relinking would log the agent out. +var errCopilotBridgeSharedEmpty = errors.New("private copilot config holds a login and the shared config holds none; leaving the private copy in place") + +// copilotConfigBridgePaths returns the agent's bridged config path and the +// shared target it must link to. ok is false when the agent has no per-agent +// home, i.e. exactly when setupInteractiveHome provisions no bridge. +func copilotConfigBridgePaths(agentName string, uid int, backend string) (link, target string, ok bool) { + if uid <= 0 || IsInferenceBackend(backend) || sharedAgentHomeForced() { + return "", "", false + } + link = filepath.Join(interactiveHomePath(agentName), copilotHomeDirName, copilotConfigFileName) + return link, sharedCopilotBridgeTarget(), true +} + +// sharedCopilotBridgeTarget is the shared config every bridge links to, +// derived from sharedAgentHome like the links themselves. +func sharedCopilotBridgeTarget() string { + return filepath.Join(sharedAgentHome, copilotHomeDirName, copilotConfigFileName) +} + +// copilotConfigDiverged reports whether the file the agent's Copilot CLI +// actually reads at link is a private regular file that no longer follows the +// shared login: unreadable, holding no usable token, or different from the +// shared file. A symlink, an absent file, or a path that IS the shared file is +// not diverged. +func copilotConfigDiverged(link, target string) bool { + info, err := os.Lstat(link) + if err != nil || !info.Mode().IsRegular() { + return false + } + if same, serr := claudeBridgeLinkIsTarget(link, target); serr != nil || same { + return false + } + private, err := sharedfile.ReadNoFollow(link, copilotConfigMaxBytes) + if err != nil { + return true + } + if !copilotCredentialFileHasTokens(link) { + return true + } + shared, err := sharedfile.ReadNoFollow(target, copilotConfigMaxBytes) + if err != nil { + // No shared file to compare against: the private copy with a token is + // the only login this agent has, and it is the one it reads. + return false + } + return string(private) != string(shared) +} + +// repairCopilotConfigBridge moves a private regular-file config.json aside and +// restores the symlink to the shared file. It returns the backup path, or "" +// with a nil error when there is nothing to repair (a symlink, an absent path, +// a directory). +func repairCopilotConfigBridge(link, target string) (string, error) { + copilotBridgeMu.Lock() + defer copilotBridgeMu.Unlock() + + info, err := os.Lstat(link) + if err != nil || !info.Mode().IsRegular() { + return "", nil + } + same, err := claudeBridgeLinkIsTarget(link, target) + if err != nil { + return "", nil + } + if same { + return "", fmt.Errorf("%w: %s", errCopilotBridgeLinkIsTarget, link) + } + if copilotCredentialFileHasTokens(link) && !copilotCredentialFileHasTokens(target) { + return "", errCopilotBridgeSharedEmpty + } + backup, err := moveAsideCopilotConfig(link, copilotBridgeNow()) + if err != nil { + return "", fmt.Errorf("move private copilot config aside: %w", err) + } + pruneCopilotConfigBackups(link, backup) + if err := os.Symlink(target, link); err != nil { + // EEXIST: the CLI saved again between the rename and now. The next + // launch repairs that one. + return backup, fmt.Errorf("restore copilot config symlink: %w", err) + } + return backup, nil +} + +// moveAsideCopilotConfig renames path to a free .bak-[-N] +// beside it. A rename keeps the file's owner and mode. +func moveAsideCopilotConfig(path string, now time.Time) (string, error) { + base := path + copilotConfigBackupInfix + now.UTC().Format("20060102T150405Z") + for i := 0; i < copilotConfigBackupNameAttempts; i++ { + dst := base + if i > 0 { + dst = fmt.Sprintf("%s-%d", base, i) + } + if _, err := os.Lstat(dst); err == nil { + continue + } else if !errors.Is(err, fs.ErrNotExist) { + return "", err + } + if err := os.Rename(path, dst); err != nil { + return "", err + } + return dst, nil + } + return "", fmt.Errorf("no free backup name for %s", path) +} + +// pruneCopilotConfigBackups removes every earlier config.json.bak-* regular +// file beside link except keep, so repeated repairs never accumulate copies of +// stale logins. +func pruneCopilotConfigBackups(link, keep string) { + matches, err := filepath.Glob(link + copilotConfigBackupInfix + "*") + if err != nil { + return + } + for _, p := range matches { + if p == keep { + continue + } + if info, lerr := os.Lstat(p); lerr == nil && info.Mode().IsRegular() { + _ = os.Remove(p) + } + } +} + +// rebridgeCopilotConfig runs the repair for one agent and logs the outcome. It +// reports whether the link was restored. +func (m *Manager) rebridgeCopilotConfig(agentName, link, target, trigger string) bool { + backup, err := repairCopilotConfigBridge(link, target) + if err != nil { + if prev, loaded := copilotBridgeWarned.Swap(link, err.Error()); loaded && prev == err.Error() { + return false + } + m.logger.Warn("copilot config bridge: repair not applied", + "agent", agentName, "link", link, "shared", target, "trigger", trigger, "error", err) + return false + } + copilotBridgeWarned.Delete(link) + if backup == "" { + return false + } + m.logger.Warn("copilot config bridge: a private config.json had replaced the shared-login symlink; re-linked", + "agent", agentName, + "trigger", trigger, + "backup", backup, + "cause", "Copilot CLI saves its config with a temp file renamed over the link", + "see", "hivecommons/hive#18443") + return true +} + +// repairCopilotConfigOnLoginPrompt is the poller's hook: a Copilot agent stuck +// at a login prompt whose config.json diverged from the shared login is +// relinked, so the credential probe then evaluates the shared login and the +// token restart that follows starts a CLI that reads it. +func (m *Manager) repairCopilotConfigOnLoginPrompt(agent *AgentProcess) bool { + m.mu.RLock() + name, uid, backend := agent.Name, agent.UID, effectiveBackend(agent) + m.mu.RUnlock() + if backend != "copilot" { + return false + } + link, target, ok := copilotConfigBridgePaths(name, uid, backend) + if !ok || !copilotConfigDiverged(link, target) { + return false + } + return m.rebridgeCopilotConfig(name, link, target, copilotBridgeTriggerLoginPrompt) +} + +// copilotConfigTmpMaxAge returns the sweep threshold: the env override when it +// parses to a positive duration, the default otherwise. +func copilotConfigTmpMaxAge() time.Duration { + if v := strings.TrimSpace(os.Getenv(copilotConfigTmpMaxAgeEnv)); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + } + return defaultCopilotConfigTmpMaxAge +} + +// sweepStaleCopilotConfigTmp removes leftover config.json.tmp.* files older +// than copilotConfigTmpMaxAge from the shared .copilot directory and the +// agent's own. They are the debris of a Copilot CLI save that died between +// writing the temp file and renaming it; a younger one may belong to a save in +// progress and is kept. Bounded and best-effort; only regular files are +// removed, and contents are never read. +func (m *Manager) sweepStaleCopilotConfigTmp(agentName, home string) { + maxAge := copilotConfigTmpMaxAge() + cutoff := copilotBridgeNow().Add(-maxAge) + prefix := copilotConfigFileName + copilotConfigTmpInfix + removed := 0 + for _, dir := range []string{filepath.Join(sharedAgentHome, copilotHomeDirName), filepath.Join(home, copilotHomeDirName)} { + entries, err := os.ReadDir(dir) + if err != nil { + continue + } + for _, e := range entries { + if removed >= copilotConfigTmpSweepCap { + break + } + if !strings.HasPrefix(e.Name(), prefix) { + continue + } + p := filepath.Join(dir, e.Name()) + info, err := os.Lstat(p) + if err != nil || !info.Mode().IsRegular() || info.ModTime().After(cutoff) { + continue + } + if err := os.Remove(p); err == nil { + removed++ + } + } + } + if removed > 0 { + m.logger.Info(fmt.Sprintf("swept %d stale copilot config tmp file(s)", removed), + "agent", agentName, "max_age", maxAge.String()) + } +} diff --git a/src/crux/agent/copilot_config_bridge_test.go b/src/crux/agent/copilot_config_bridge_test.go new file mode 100644 index 0000000000..3ec8bee24a --- /dev/null +++ b/src/crux/agent/copilot_config_bridge_test.go @@ -0,0 +1,489 @@ +package agent + +import ( + "bytes" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/config" +) + +// Fake Copilot configs. Every token value carries this sentinel so a test can +// prove no log line ever leaks token material. No real token appears here. +const copilotBridgeTestSentinel = "FAKE-COPILOT-SENTINEL" + +const ( + copilotBridgeTestLoggedIn = `{"copilotTokens":{"https://github.com:fake-user":"gho_` + copilotBridgeTestSentinel + `"}}` + copilotBridgeTestOlder = `{"copilotTokens":{"https://github.com:fake-user":"gho_older_` + copilotBridgeTestSentinel + `"}}` + copilotBridgeTestNoLogin = `{"copilotTokens":{}}` + copilotBridgeTestAgentName = "scanner" + copilotBridgeTestUID = 1001 +) + +// copilotBridgeFixture provisions a per-agent home for a copilot agent with the +// shared login in place and config.json bridged. It returns the manager, the +// log buffer, the agent, the link and the shared target. +func copilotBridgeFixture(t *testing.T) (*Manager, *bytes.Buffer, *AgentProcess, string, string) { + t.Helper() + shared := withSharedAgentHome(t) + withCopilotBridgeClock(t, time.Date(2026, 11, 16, 10, 0, 0, 0, time.UTC)) + copilotBridgeWarned.Delete(filepath.Join(interactiveHomePath(copilotBridgeTestAgentName), copilotHomeDirName, copilotConfigFileName)) + var buf bytes.Buffer + m := NewManager(map[string]config.AgentConfig{}, slog.New(slog.NewTextHandler(&buf, nil)), ProjectContext{}) + ap := &AgentProcess{Name: copilotBridgeTestAgentName, UID: copilotBridgeTestUID, Config: config.AgentConfig{Backend: "copilot"}} + m.mu.Lock() + m.agents[ap.Name] = ap + m.mu.Unlock() + target := filepath.Join(shared, copilotHomeDirName, copilotConfigFileName) + writeCopilotBridgeFile(t, target, copilotBridgeTestLoggedIn) + m.setupInteractiveHome(ap, "copilot") + link := filepath.Join(interactiveHomePath(ap.Name), copilotHomeDirName, copilotConfigFileName) + assertCopilotLinked(t, link, target) + return m, &buf, ap, link, target +} + +func withCopilotBridgeClock(t *testing.T, now time.Time) { + t.Helper() + old := copilotBridgeNow + copilotBridgeNow = func() time.Time { return now } + t.Cleanup(func() { copilotBridgeNow = old }) +} + +func writeCopilotBridgeFile(t *testing.T, path, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } +} + +// simulateCopilotSave does what the Copilot CLI's config store does: write a +// temp file beside config.json and rename it over it, replacing the symlink. +func simulateCopilotSave(t *testing.T, link, body string) { + t.Helper() + tmp := link + copilotConfigTmpInfix + "abc123" + if err := os.WriteFile(tmp, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Rename(tmp, link); err != nil { + t.Fatal(err) + } + if info, err := os.Lstat(link); err != nil || !info.Mode().IsRegular() { + t.Fatalf("simulated save did not replace the link: info=%v err=%v", info, err) + } +} + +func assertCopilotLinked(t *testing.T, link, target string) { + t.Helper() + got, err := os.Readlink(link) + if err != nil || got != target { + t.Fatalf("copilot config bridge: readlink=%q err=%v, want %q", got, err, target) + } +} + +func copilotBackups(t *testing.T, link string) []string { + t.Helper() + matches, err := filepath.Glob(link + copilotConfigBackupInfix + "*") + if err != nil { + t.Fatal(err) + } + return matches +} + +func assertNoSentinelLogged(t *testing.T, buf *bytes.Buffer) { + t.Helper() + if strings.Contains(buf.String(), copilotBridgeTestSentinel) { + t.Fatalf("log leaked token material:\n%s", buf.String()) + } +} + +func TestCopilotConfigBridge_RestartRelinksEmptyPrivateCopy(t *testing.T) { + m, buf, ap, link, target := copilotBridgeFixture(t) + simulateCopilotSave(t, link, copilotBridgeTestNoLogin) + + m.setupInteractiveHome(ap, "copilot") + + assertCopilotLinked(t, link, target) + backups := copilotBackups(t, link) + if len(backups) != 1 { + t.Fatalf("want exactly one backup, got %v", backups) + } + if want := link + copilotConfigBackupInfix + "20261116T100000Z"; backups[0] != want { + t.Fatalf("backup = %q, want %q", backups[0], want) + } + data, err := os.ReadFile(backups[0]) + if err != nil || string(data) != copilotBridgeTestNoLogin { + t.Fatalf("backup must hold the private copy unchanged: data=%q err=%v", data, err) + } + if shared, err := os.ReadFile(target); err != nil || string(shared) != copilotBridgeTestLoggedIn { + t.Fatalf("shared login must not change: err=%v", err) + } + if !strings.Contains(buf.String(), "re-linked") { + t.Fatalf("repair not logged:\n%s", buf.String()) + } + assertNoSentinelLogged(t, buf) +} + +func TestCopilotConfigBridge_ValidSymlinkLeftAlone(t *testing.T) { + m, buf, ap, link, target := copilotBridgeFixture(t) + before, err := os.Lstat(link) + if err != nil { + t.Fatal(err) + } + m.setupInteractiveHome(ap, "copilot") + after, err := os.Lstat(link) + if err != nil { + t.Fatal(err) + } + assertCopilotLinked(t, link, target) + if !os.SameFile(before, after) { + t.Fatal("a valid symlink must not be replaced") + } + if b := copilotBackups(t, link); len(b) != 0 { + t.Fatalf("no backup expected for a valid link, got %v", b) + } + if strings.Contains(buf.String(), "re-linked") { + t.Fatalf("a valid link must not be reported as repaired:\n%s", buf.String()) + } +} + +func TestCopilotConfigBridge_RepeatedRestartsConvergeAndKeepNewestBackup(t *testing.T) { + m, buf, ap, link, target := copilotBridgeFixture(t) + base := time.Date(2026, 11, 16, 11, 0, 0, 0, time.UTC) + var last string + for i := 0; i < 3; i++ { + now := base.Add(time.Duration(i) * time.Minute) + withCopilotBridgeClock(t, now) + simulateCopilotSave(t, link, copilotBridgeTestNoLogin) + m.setupInteractiveHome(ap, "copilot") + assertCopilotLinked(t, link, target) + last = link + copilotConfigBackupInfix + now.UTC().Format("20060102T150405Z") + } + // Further restarts with nothing to repair change nothing. + m.setupInteractiveHome(ap, "copilot") + m.setupInteractiveHome(ap, "copilot") + assertCopilotLinked(t, link, target) + backups := copilotBackups(t, link) + if len(backups) != 1 || backups[0] != last { + t.Fatalf("want only the newest backup %q, got %v", last, backups) + } + assertNoSentinelLogged(t, buf) +} + +func TestCopilotConfigBridge_PrivateLoginKeptWhenSharedEmpty(t *testing.T) { + m, buf, ap, link, target := copilotBridgeFixture(t) + simulateCopilotSave(t, link, copilotBridgeTestLoggedIn) + writeCopilotBridgeFile(t, target, copilotBridgeTestNoLogin) + + m.setupInteractiveHome(ap, "copilot") + m.setupInteractiveHome(ap, "copilot") + + info, err := os.Lstat(link) + if err != nil || !info.Mode().IsRegular() { + t.Fatalf("private copy holding the only login must stay in place: info=%v err=%v", info, err) + } + if b := copilotBackups(t, link); len(b) != 0 { + t.Fatalf("no backup expected, got %v", b) + } + if got := strings.Count(buf.String(), "repair not applied"); got != 1 { + t.Fatalf("refusal must be logged once, got %d:\n%s", got, buf.String()) + } + assertNoSentinelLogged(t, buf) +} + +func TestRepairCopilotConfigBridge_LegacyDirectoryLinkUntouched(t *testing.T) { + shared := withSharedAgentHome(t) + target := filepath.Join(shared, copilotHomeDirName, copilotConfigFileName) + writeCopilotBridgeFile(t, target, copilotBridgeTestLoggedIn) + home := filepath.Join(t.TempDir(), "home") + if err := os.MkdirAll(home, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Dir(target), filepath.Join(home, copilotHomeDirName)); err != nil { + t.Fatal(err) + } + link := filepath.Join(home, copilotHomeDirName, copilotConfigFileName) + + backup, err := repairCopilotConfigBridge(link, target) + if !errors.Is(err, errCopilotBridgeLinkIsTarget) || backup != "" { + t.Fatalf("want errCopilotBridgeLinkIsTarget and no backup, got %q, %v", backup, err) + } + if data, rerr := os.ReadFile(target); rerr != nil || string(data) != copilotBridgeTestLoggedIn { + t.Fatalf("shared login must be untouched: err=%v", rerr) + } + if copilotConfigDiverged(link, target) { + t.Fatal("a path that IS the shared file is not diverged") + } +} + +func TestRepairCopilotConfigBridge_NothingToRepair(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "shared.json") + // Absent link. + if b, err := repairCopilotConfigBridge(filepath.Join(dir, "absent.json"), target); b != "" || err != nil { + t.Fatalf("absent link: got %q, %v", b, err) + } + // A directory at the link path is not this failure. + d := filepath.Join(dir, "adir") + if err := os.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } + if b, err := repairCopilotConfigBridge(d, target); b != "" || err != nil { + t.Fatalf("directory link: got %q, %v", b, err) + } + // Unresolvable directories: nothing is moved. + link := filepath.Join(dir, "config.json") + writeCopilotBridgeFile(t, link, copilotBridgeTestNoLogin) + missingTarget := filepath.Join(dir, "missing", "config.json") + if b, err := repairCopilotConfigBridge(link, missingTarget); b != "" || err != nil { + t.Fatalf("unresolvable target dir: got %q, %v", b, err) + } + if info, err := os.Lstat(link); err != nil || !info.Mode().IsRegular() { + t.Fatalf("file must stay in place: info=%v err=%v", info, err) + } +} + +func TestMoveAsideCopilotConfig_PicksFreeName(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, copilotConfigFileName) + now := time.Date(2026, 11, 16, 12, 0, 0, 0, time.UTC) + taken := path + copilotConfigBackupInfix + "20261116T120000Z" + writeCopilotBridgeFile(t, taken, copilotBridgeTestNoLogin) + writeCopilotBridgeFile(t, path, copilotBridgeTestNoLogin) + + dst, err := moveAsideCopilotConfig(path, now) + if err != nil { + t.Fatal(err) + } + if want := taken + "-1"; dst != want { + t.Fatalf("dst = %q, want %q", dst, want) + } + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatalf("source must be gone after the move: %v", err) + } + if _, err := moveAsideCopilotConfig(filepath.Join(dir, "absent"), now); err == nil { + t.Fatal("moving an absent file must fail") + } +} + +func TestPruneCopilotConfigBackups_KeepsOnlyNamedAndNonRegular(t *testing.T) { + dir := t.TempDir() + link := filepath.Join(dir, copilotConfigFileName) + keep := link + copilotConfigBackupInfix + "b" + old := link + copilotConfigBackupInfix + "a" + sub := link + copilotConfigBackupInfix + "dir" + writeCopilotBridgeFile(t, keep, "{}") + writeCopilotBridgeFile(t, old, "{}") + if err := os.Mkdir(sub, 0o755); err != nil { + t.Fatal(err) + } + pruneCopilotConfigBackups(link, keep) + if _, err := os.Lstat(keep); err != nil { + t.Fatalf("kept backup removed: %v", err) + } + if _, err := os.Lstat(old); !os.IsNotExist(err) { + t.Fatalf("older backup must be pruned: %v", err) + } + if _, err := os.Lstat(sub); err != nil { + t.Fatalf("non-regular entry must be left alone: %v", err) + } +} + +func TestCopilotConfigDiverged(t *testing.T) { + dir := t.TempDir() + sharedDir := filepath.Join(dir, "shared") + agentDir := filepath.Join(dir, "agent") + target := filepath.Join(sharedDir, copilotConfigFileName) + link := filepath.Join(agentDir, copilotConfigFileName) + writeCopilotBridgeFile(t, target, copilotBridgeTestLoggedIn) + if err := os.MkdirAll(agentDir, 0o755); err != nil { + t.Fatal(err) + } + + if copilotConfigDiverged(link, target) { + t.Fatal("absent file is not diverged") + } + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + if copilotConfigDiverged(link, target) { + t.Fatal("a symlink is not diverged") + } + _ = os.Remove(link) + + cases := []struct { + name string + private string + want bool + }{ + {"empty login", copilotBridgeTestNoLogin, true}, + {"unparseable", "not json", true}, + {"differs from shared", copilotBridgeTestOlder, true}, + {"identical to shared", copilotBridgeTestLoggedIn, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + writeCopilotBridgeFile(t, link, tc.private) + if got := copilotConfigDiverged(link, target); got != tc.want { + t.Fatalf("diverged = %v, want %v", got, tc.want) + } + }) + } + + // No shared file to compare: a private copy with a login is what the + // agent reads and is not diverged. + if err := os.Remove(target); err != nil { + t.Fatal(err) + } + writeCopilotBridgeFile(t, link, copilotBridgeTestOlder) + if copilotConfigDiverged(link, target) { + t.Fatal("private login with no shared file is not diverged") + } +} + +func TestCopilotConfigBridgePaths(t *testing.T) { + withSharedAgentHome(t) + if _, _, ok := copilotConfigBridgePaths("a", 0, "copilot"); ok { + t.Fatal("uid 0 has no per-agent bridge") + } + if _, _, ok := copilotConfigBridgePaths("a", copilotBridgeTestUID, "litellm"); ok { + t.Fatal("inference backends have no per-agent bridge") + } + t.Setenv("HIVE_SHARED_AGENT_HOME", "1") + if _, _, ok := copilotConfigBridgePaths("a", copilotBridgeTestUID, "copilot"); ok { + t.Fatal("forced shared home has no per-agent bridge") + } +} + +// The health check must fail when the per-agent file differs from the shared +// one, even though the shared file and the in-memory token both hold a login. +func TestCredentialProbe_CopilotEvaluatesFileAgentReads(t *testing.T) { + m, buf, ap, link, target := copilotBridgeFixture(t) + m.mu.Lock() + m.copilotAuthToken = "fake-in-memory-" + copilotBridgeTestSentinel + m.mu.Unlock() + + if !m.AgentHasValidCredential(ap.Name) { + t.Fatal("linked config with a shared login must prove the credential") + } + simulateCopilotSave(t, link, copilotBridgeTestNoLogin) + if m.AgentHasValidCredential(ap.Name) { + t.Fatal("a diverged per-agent copy must not count as a valid credential") + } + simulateCopilotSave(t, link, copilotBridgeTestOlder) + if m.AgentHasValidCredential(ap.Name) { + t.Fatal("a per-agent copy that differs from the shared login must not count as valid") + } + + // The poller's login-prompt hook repairs it, and the probe then passes. + if !m.repairCopilotConfigOnLoginPrompt(ap) { + t.Fatal("login-prompt hook must repair a diverged copy") + } + assertCopilotLinked(t, link, target) + if !m.AgentHasValidCredential(ap.Name) { + t.Fatal("after relinking, the shared login must prove the credential") + } + if m.repairCopilotConfigOnLoginPrompt(ap) { + t.Fatal("a second call has nothing to repair") + } + assertNoSentinelLogged(t, buf) +} + +func TestRepairCopilotConfigOnLoginPrompt_IgnoresOtherBackends(t *testing.T) { + m, _, _, link, _ := copilotBridgeFixture(t) + simulateCopilotSave(t, link, copilotBridgeTestNoLogin) + other := &AgentProcess{Name: copilotBridgeTestAgentName, UID: copilotBridgeTestUID, Config: config.AgentConfig{Backend: "claude"}} + if m.repairCopilotConfigOnLoginPrompt(other) { + t.Fatal("non-copilot backends are not repaired by the copilot hook") + } + if info, err := os.Lstat(link); err != nil || !info.Mode().IsRegular() { + t.Fatalf("file must stay in place: info=%v err=%v", info, err) + } + noHome := &AgentProcess{Name: "planner", Config: config.AgentConfig{Backend: "copilot"}} + if m.repairCopilotConfigOnLoginPrompt(noHome) { + t.Fatal("agent without a per-agent home has nothing to repair") + } +} + +func TestCopilotConfigTmpMaxAge(t *testing.T) { + t.Setenv(copilotConfigTmpMaxAgeEnv, "") + if got := copilotConfigTmpMaxAge(); got != defaultCopilotConfigTmpMaxAge { + t.Fatalf("default = %v", got) + } + t.Setenv(copilotConfigTmpMaxAgeEnv, "15m") + if got := copilotConfigTmpMaxAge(); got != 15*time.Minute { + t.Fatalf("override = %v", got) + } + for _, bad := range []string{"nonsense", "-5m", "0s"} { + t.Setenv(copilotConfigTmpMaxAgeEnv, bad) + if got := copilotConfigTmpMaxAge(); got != defaultCopilotConfigTmpMaxAge { + t.Fatalf("invalid %q must fall back to default, got %v", bad, got) + } + } +} + +func TestSweepStaleCopilotConfigTmp(t *testing.T) { + shared := withSharedAgentHome(t) + now := time.Date(2026, 11, 16, 12, 0, 0, 0, time.UTC) + withCopilotBridgeClock(t, now) + t.Setenv(copilotConfigTmpMaxAgeEnv, "") + var buf bytes.Buffer + m := &Manager{logger: slog.New(slog.NewTextHandler(&buf, nil))} + + home := filepath.Join(t.TempDir(), "agent") + sharedDir := filepath.Join(shared, copilotHomeDirName) + agentDir := filepath.Join(home, copilotHomeDirName) + staleShared := filepath.Join(sharedDir, copilotConfigFileName+copilotConfigTmpInfix+"old1") + staleAgent := filepath.Join(agentDir, copilotConfigFileName+copilotConfigTmpInfix+"old2") + fresh := filepath.Join(sharedDir, copilotConfigFileName+copilotConfigTmpInfix+"new") + unrelated := filepath.Join(sharedDir, "other.json.tmp.x") + tmpDir := filepath.Join(sharedDir, copilotConfigFileName+copilotConfigTmpInfix+"dir") + for _, p := range []string{staleShared, staleAgent, fresh, unrelated} { + writeCopilotBridgeFile(t, p, "{}") + } + if err := os.Mkdir(tmpDir, 0o755); err != nil { + t.Fatal(err) + } + old := now.Add(-2 * defaultCopilotConfigTmpMaxAge) + for _, p := range []string{staleShared, staleAgent, unrelated, tmpDir} { + if err := os.Chtimes(p, old, old); err != nil { + t.Fatal(err) + } + } + young := now.Add(-time.Minute) + if err := os.Chtimes(fresh, young, young); err != nil { + t.Fatal(err) + } + + m.sweepStaleCopilotConfigTmp("scanner", home) + + for _, gone := range []string{staleShared, staleAgent} { + if _, err := os.Lstat(gone); !os.IsNotExist(err) { + t.Fatalf("%s should be swept: %v", gone, err) + } + } + for _, kept := range []string{fresh, unrelated, tmpDir} { + if _, err := os.Lstat(kept); err != nil { + t.Fatalf("%s should be kept: %v", kept, err) + } + } + if !strings.Contains(buf.String(), "swept 2 stale copilot config tmp file(s)") { + t.Fatalf("sweep not logged:\n%s", buf.String()) + } + + // A shorter threshold from the env sweeps the younger file too. + t.Setenv(copilotConfigTmpMaxAgeEnv, "30s") + m.sweepStaleCopilotConfigTmp("scanner", home) + if _, err := os.Lstat(fresh); !os.IsNotExist(err) { + t.Fatalf("env threshold not honoured: %v", err) + } + // Missing directories are not an error. + m.sweepStaleCopilotConfigTmp("scanner", filepath.Join(t.TempDir(), "nohome")) +} diff --git a/src/crux/agent/interactive_home.go b/src/crux/agent/interactive_home.go index 4b9798400d..0a54771f39 100644 --- a/src/crux/agent/interactive_home.go +++ b/src/crux/agent/interactive_home.go @@ -279,6 +279,7 @@ func (m *Manager) setupInteractiveHome(agent *AgentProcess, backend string) { m.seedClaudeSessionForAgent(agent, home) m.tightenInteractiveHome(agent.Name, home, agent.UID) m.sweepOrphanedClaudeTmp(agent.Name) + m.sweepStaleCopilotConfigTmp(agent.Name, home) m.warnOnStrayGitIdentity(agent.Name, home) } @@ -431,6 +432,13 @@ func (m *Manager) setupPerAgentCLIHome(agentName, home string, uid int, spec per // bridgeHomeEntry, which refuses to clobber a regular file. m.rebridgeClaudeCredential(agentName, link, target, claudeBridgeTriggerLaunch) } + if spec.dir == copilotHomeDirName && name == copilotConfigFileName { + // The Copilot CLI's atomic save replaced the link with a private + // file that no longer follows the shared login (#18443); move it + // aside and relink BEFORE bridgeHomeEntry, which refuses to clobber + // a regular file. + m.rebridgeCopilotConfig(agentName, link, target, copilotBridgeTriggerLaunch) + } if _, err := os.Stat(target); err != nil && isPerAgentCLIBridgeDir(spec.dir, name) { // Absent target: a dangling FILE link goes live on first login; a // dangling DIRECTORY link would make the CLI's own mkdir -p fail, diff --git a/src/crux/agent/manager_poll.go b/src/crux/agent/manager_poll.go index e853703e24..f14a0dcae3 100644 --- a/src/crux/agent/manager_poll.go +++ b/src/crux/agent/manager_poll.go @@ -172,6 +172,14 @@ func (m *Manager) pollTmuxOutputForAgent(agent *AgentProcess, ctx context.Contex // 5. A quiet-output window: a genuinely stuck login prompt sits // silent, while an agent still rendering output may simply be // discussing login/auth work in its answer. + // + // A Copilot agent whose config.json diverged from the shared login + // (#18443) is relinked first: the credential probe below rejects a + // diverged copy, and once relinked it evaluates the shared login, so + // the one restart that follows starts a CLI that reads it. + if showsLogin && loginStreak >= loginStreakRestartMin { + m.repairCopilotConfigOnLoginPrompt(agent) + } if showsLogin && loginStreak >= loginStreakRestartMin && m.AgentHasValidCredential(agent.Name) { now := time.Now() m.mu.RLock() diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index a862499e65..85c2ed2cb6 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -121,6 +121,7 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_DELEGATION_CHAIN_ENABLED` | No | disabled | Enables delegation chain minting (`1`, `true`, `yes`, `on` — same spelling as `HIVE_METRICS_ENABLED`). Read on each call rather than cached, so disabling it on a misbehaving spoke does not require a pod roll. | | `HIVE_ALLOW_PRIVATE_GIT_SOURCE` | No | `false` | Exact `true` opt-in read by `pkg/knowledge/gitsource.go` to allow knowledge Git sources whose host resolves to a private/internal address (self-hosted GitLab and similar). Off by default as SSRF protection. | | `HIVE_SHARED_AGENT_HOME` | No | per-agent HOME | Escape hatch (`1`) restoring the legacy shared-HOME layout for agents. | +| `HIVE_COPILOT_CONFIG_TMP_MAX_AGE` | No | `1h` | Age (Go duration) a leftover Copilot CLI `config.json.tmp.*` save file must reach before the agent-launch sweep removes it from the shared and per-agent `~/.copilot` directories. Invalid or non-positive values fall back to the default. See [the Copilot login troubleshooting entry](troubleshooting.md#one-copilot-agent-stays-at-the-login-prompt-after-a-dashboard-login). | | `HIVE_WORKSPACE_CLEANUP_ENABLED` | No | enabled | Set `0` to opt out of automatic agent workspace cleanup. | | `HIVE_GITHUB_OWNER_TOKEN_FILE` | No | `/data/secrets/github-owner-token` | Where the spoke keeps the hive owner's GitHub token for write-tier reference repos (`src/pkg/github/owner_identity.go`, #18197): an owner-only JSON file written by the Settings consent flow, never by hand and never in `hive.yaml`. Move it only together with the secrets volume. | | `HIVE_REMOTE_ITEMS_FILE` | No | `/data/turn/remote-items.json` | The tracker for issues and PRs the hive opened in write-tier reference repos (`src/pkg/remoteitems`, #18199): cursors per item and the feedback queued for an agent's next kick. | diff --git a/src/docs/troubleshooting.md b/src/docs/troubleshooting.md index b6ecc797a3..ce9e73e08b 100644 --- a/src/docs/troubleshooting.md +++ b/src/docs/troubleshooting.md @@ -29,7 +29,7 @@ legacy v1 host install and do not apply here. | Need logs without host access | [No-shell route](#no-shell-on-the-host) | Admin MCP is **v6-only**, not assumed available on v5 | | Notifications absent | [Notification checks](#notifications-ntfy--slack--discord-never-arrive) | Dashboard operator; outbound-network test needs container shell | | Agent committed but no remote branch/PR | [Publication diagnosis](#an-agent-session-completes-but-no-branch-or-pr-appears) | Container shell, with the agent's UID/HOME | -| Login repeats or Copilot licensing fails | [Login loop](#an-agent-says-please-run-login-but-logging-in-changes-nothing), [license check](#every-copilot-agent-says-you-are-not-licensed-to-use-copilot) | Backend account owner; container admin for file-access repair | +| Login repeats or Copilot licensing fails | [Login loop](#an-agent-says-please-run-login-but-logging-in-changes-nothing), [license check](#every-copilot-agent-says-you-are-not-licensed-to-use-copilot), [one Copilot agent ignores the login](#one-copilot-agent-stays-at-the-login-prompt-after-a-dashboard-login) | Backend account owner; container admin for file-access repair | | Browser terminal frozen | [Terminal recovery](#the-terminal-looks-frozen--no-new-output-and-reopening-it-doesnt-help) | Browser/dashboard first; container shell only for deeper diagnosis | | Agent behavior/model unexpected | [Reasoning logs](#an-agent-runs-but-not-the-way-i-expect--why-did-it-do-that), [model selection](#switching-an-agents-model) | Dashboard operator | | Dashboard access or API startup delays | [Auth/access](#dashboard-auth-and-access-problems), [startup response](#the-api-says-service-starting-up-please-retry), [health endpoints](#health-endpoints) | Dashboard owner or deployment admin, according to the failing layer | @@ -572,6 +572,58 @@ a login prompt: it badged the agent 🔑, and — because a valid token was on d auto-restarted it straight back into the same 403, which looked like the agent crash-looping. Hive no longer treats a 403 as a login signal; a 401 still is. +## One Copilot agent stays at the login prompt after a dashboard login + +Symptom: the dashboard's Copilot device-flow login succeeds and every other +Copilot agent keeps working, but one agent stays at the Copilot CLI login +prompt. Before the fix for +[#18443](https://github.com/hivecommons/hive/issues/18443) the hive restarted +it a few times and then gave up (`giving up on token-triggered restart`). + +Cause: each per-agent `~/.copilot/config.json` is a symlink to the shared +`/data/home/.copilot/config.json`, which is where the dashboard login and the +token sync write. The Copilot CLI saves its config by writing a temp file and +renaming it over `config.json`. That replaces the symlink with a private +regular file, and from then on the agent never sees a new login. An agent whose +private copy holds no login sits at the prompt. Agents whose private copy still +holds an older token keep working until that token expires. A leftover +`config.json.tmp.*` in the shared `.copilot` folder is the trace of such a save. + +Automatic repair: at every agent launch and restart, the hive checks the +agent's `~/.copilot/config.json`. When it is a regular file instead of the link, +the hive moves it aside to `config.json.bak-` (keeping only the +newest backup) and links it back to the shared file. The log line is +`copilot config bridge: a private config.json had replaced the shared-login symlink; re-linked`. +The credential check behind the token-triggered restart now looks at the file +the agent actually reads: a private copy that holds no login, or that differs +from the shared file, does not count as logged in. When the pane poller sees a +Copilot agent stuck at the login prompt with such a copy, it relinks it first, +so the one restart that follows starts a CLI that reads the shared login. File +contents are never logged. + +The repair leaves the private copy in place when it holds a login and the +shared file holds none (it may be the only login the hive has); the log says +`copilot config bridge: repair not applied`. Sign in from the dashboard to fill +the shared file and the next launch relinks the agent. + +Launch also removes leftover `config.json.tmp.*` files older than one hour from +the shared and per-agent `.copilot` folders; younger ones may belong to a save +in progress. Set `HIVE_COPILOT_CONFIG_TMP_MAX_AGE` (a Go duration such as +`30m`) to change the threshold. + +Manual workaround (hives without the fix, or to force it now): inside the hive +container, move the agent's private copy aside, then restart only that agent. + +```bash +mv /data/home/agents//.copilot/config.json \ + /data/home/agents//.copilot/config.json.bak-manual +curl -X POST -H "Authorization: Bearer $HIVE_DASHBOARD_TOKEN" \ + http://localhost:3002/api/restart/ +``` + +The restart recreates the link to the shared login. Do not open or copy the +config file's contents: it holds the Copilot token. + ## Every Copilot agent says "You are not licensed to use Copilot" ```