diff --git a/changelog.d/fixed-agentsmd-hosted-fallback.md b/changelog.d/fixed-agentsmd-hosted-fallback.md new file mode 100644 index 0000000000..6b70e6c3ed --- /dev/null +++ b/changelog.d/fixed-agentsmd-hosted-fallback.md @@ -0,0 +1 @@ +- Hosted spokes now inject the primary repo's `AGENTS.md` into kicks: when neither `project.checkouts_dir` nor a matching `policies.local_dir` names a local checkout, the scheduler fetches `AGENTS.md` from the repo's default branch through the hive's GitHub client (15-minute cache, last-known-good on API failure, same injection cap) and materialises it under `/data/agentsmd-cache///` so the log shows `agentsmd: injecting repo instructions into kick` with that root ([#17090](https://github.com/hivecommons/hive/issues/17090)). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index e9f1479ed2..9f3d24df71 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1489,6 +1489,15 @@ func (b *boot) bootGovernor() { b.logger, )) + // Hosted spokes keep no checkout of their primary repo, so the two local + // roots agentsRepoRoot knows are empty there and AGENTS.md was never + // injected (hivecommons/hive#17090). Let the scheduler fetch it through the + // same App client when no local root exists. Guarded so a typed-nil + // *github.Client is never stored as a non-nil interface. + if b.ghClient != nil { + b.sched.SetAgentsMdFetcher(b.ghClient) + } + // Wire the whole-agent definition_source resolver so agents imported with // "keep linked" re-fetch their portable AgentDefinition from the repo on // reload/kick and re-apply its operator-safe fields (never security/seed-only diff --git a/src/pkg/scheduler/kick_agentsmd_remote.go b/src/pkg/scheduler/kick_agentsmd_remote.go new file mode 100644 index 0000000000..97a600ebf3 --- /dev/null +++ b/src/pkg/scheduler/kick_agentsmd_remote.go @@ -0,0 +1,158 @@ +package scheduler + +import ( + "context" + "os" + "path/filepath" + "strings" + "time" + + "github.com/hivecommons/hive/crux/agentsmd" + "github.com/hivecommons/hive/pkg/config" +) + +// AgentsMdFetcher is the one GitHub call the remote AGENTS.md fallback needs. +// *github.Client satisfies it; the interface exists so tests can fake it and so +// a token-less boot can leave it unset. +type AgentsMdFetcher interface { + GetFileContentRef(ctx context.Context, owner, repo, path, ref string) (string, error) +} + +const ( + // remoteAgentsMdTTL is how long one fetched AGENTS.md serves kicks before + // the scheduler asks GitHub again. The shared transport sends + // If-None-Match, so a refresh of an unchanged file is a free 304 — the + // TTL only bounds how often that round trip happens, not quota. + remoteAgentsMdTTL = 15 * time.Minute + // remoteAgentsMdFetchTimeout bounds one fetch so a slow API cannot stall + // a kick; on timeout the last-known-good copy (if any) is served. + remoteAgentsMdFetchTimeout = 10 * time.Second +) + +// remoteAgentsMdDir is where fetched AGENTS.md files are materialised, as +// "///AGENTS.md", so agentsmd.Parse and primeSkills read it +// exactly as they would a checkout. A var only so tests can point it at a +// temp dir. +var remoteAgentsMdDir = "/data/agentsmd-cache" + +// remoteAgentsMd is one repo's fetch state. root is "" when the last fetch +// found no AGENTS.md (negative cache: a repo without the file is not asked +// about again until the TTL lapses). +type remoteAgentsMd struct { + root string + fetched time.Time +} + +// SetAgentsMdFetcher attaches the GitHub client the scheduler falls back to +// when a hive has no local checkout of its primary repo (hivecommons/hive#17090). +// Nil leaves the fallback off, which is the pre-existing behaviour. +func (s *Scheduler) SetAgentsMdFetcher(f AgentsMdFetcher) { + s.mu.Lock() + defer s.mu.Unlock() + s.agentsMdFetcher = f +} + +func (s *Scheduler) agentsMdFetcherOrNil() AgentsMdFetcher { + s.mu.RLock() + defer s.mu.RUnlock() + return s.agentsMdFetcher +} + +// remoteAgentsRepoRoot returns a local root holding repo's AGENTS.md fetched +// from the forge at its default branch, or "" when no fetcher is wired, the +// repo has no AGENTS.md, or the fetch failed with nothing cached. +// +// Hosted spokes keep no clone of the repos they work on — agents go through +// the API — so without this the two local roots agentsRepoRoot knows are +// always empty and the repo's instructions never reach a kick. The file is +// written under remoteAgentsMdDir so the rest of the priming path (Parse, +// Render, the injection cap, primeSkills' repo-local fallback) is unchanged +// and the log line's root= says where the text came from. +func (s *Scheduler) remoteAgentsRepoRoot(repo string) string { + fetcher := s.agentsMdFetcherOrNil() + if fetcher == nil || s.cfg == nil { + return "" + } + owner, name, ok := strings.Cut(config.QualifyRepo(s.cfg.Project.Org, repo), "/") + owner, name = strings.TrimSpace(owner), strings.TrimSpace(name) + if !ok || owner == "" || name == "" || !safeRepoPathSegment(owner) || !safeRepoPathSegment(name) { + return "" + } + key := owner + "/" + name + + s.remoteAgentsMu.Lock() + defer s.remoteAgentsMu.Unlock() + if s.remoteAgents == nil { + s.remoteAgents = map[string]remoteAgentsMd{} + } + now := time.Now() + cached, have := s.remoteAgents[key] + if have && now.Sub(cached.fetched) < remoteAgentsMdTTL { + return cached.root + } + + ctx, cancel := context.WithTimeout(context.Background(), remoteAgentsMdFetchTimeout) + defer cancel() + body, err := fetcher.GetFileContentRef(ctx, owner, name, agentsmd.AgentsFileName, "") + if err != nil { + if have && cached.root != "" { + // Last-known-good: a transient API failure must not strip the + // repo's instructions from a kick. Retry on the next kick. + s.logger.Warn("agentsmd: refresh from GitHub failed, serving cached copy", + "repo", key, "root", cached.root, "error", err) + return cached.root + } + // A 404 is the normal "repo has no AGENTS.md" answer; everything else + // is still just a miss for this kick, said once per TTL. + s.logger.Info("agentsmd: no AGENTS.md fetched from GitHub", "repo", key, "error", err) + s.remoteAgents[key] = remoteAgentsMd{fetched: now} + return "" + } + + root := filepath.Join(remoteAgentsMdDir, owner, name) + if err := writeFileAtomic(filepath.Join(root, agentsmd.AgentsFileName), []byte(body)); err != nil { + s.logger.Warn("agentsmd: cannot materialise fetched AGENTS.md, skipping injection", + "repo", key, "root", root, "error", err) + if have && cached.root != "" { + return cached.root + } + return "" + } + s.logger.Info("agentsmd: fetched AGENTS.md from GitHub", "repo", key, "bytes", len(body), "root", root) + s.remoteAgents[key] = remoteAgentsMd{root: root, fetched: now} + return root +} + +// safeRepoPathSegment refuses an owner or repo name that could escape +// remoteAgentsMdDir when joined into a path. Names come from config, which +// is operator-written, so this is belt-and-braces rather than a trust gate. +func safeRepoPathSegment(seg string) bool { + return seg != "." && seg != ".." && !strings.ContainsAny(seg, `/\`) +} + +// writeFileAtomic writes data to path via a sibling temp file and rename so a +// kick reading AGENTS.md concurrently never sees a half-written file. +func writeFileAtomic(path string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(path), ".agentsmd-*") + if err != nil { + return err + } + tmpName := tmp.Name() + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + _ = os.Remove(tmpName) + return err + } + if err := tmp.Close(); err != nil { + _ = os.Remove(tmpName) + return err + } + if err := os.Rename(tmpName, path); err != nil { + _ = os.Remove(tmpName) + return err + } + return nil +} diff --git a/src/pkg/scheduler/kick_priming.go b/src/pkg/scheduler/kick_priming.go index b8f4b90ccf..0417b1f233 100644 --- a/src/pkg/scheduler/kick_priming.go +++ b/src/pkg/scheduler/kick_priming.go @@ -18,7 +18,7 @@ const maxIssuesToPrime = 5 // when the hive has none — in which case primeAgentsMd is a no-op, exactly as // before this was wired. // -// Two sources, in order: +// Three sources, in order: // // 1. project.checkouts_dir — the explicit one. An operator who mounts checkouts // of the monitored repos points at the parent directory and each repo is @@ -29,8 +29,12 @@ const maxIssuesToPrime = 5 // out of it), but it is a checkout of policies.repo, so it is used ONLY when // that repo IS the repo being asked about. Otherwise it would inject the // config repo's AGENTS.md into work on an unrelated repo. +// 3. the forge — when neither local root is configured and a GitHub client is +// wired (SetAgentsMdFetcher), AGENTS.md is fetched from the repo's default +// branch and materialised under remoteAgentsMdDir (#17090). Hosted spokes +// keep no clones at all, so without this they never injected anything. // -// Neither is checked for existence here: agentsmd.Parse tolerates a missing +// Neither local root is checked for existence here: agentsmd.Parse tolerates a missing // directory or file and yields "", and primeAgentsMd logs which root came up // empty. One stat per kick to say the same thing earlier is not worth it. func (s *Scheduler) agentsRepoRoot(repo string) string { @@ -44,7 +48,7 @@ func (s *Scheduler) agentsRepoRoot(repo string) string { sameRepo(s.cfg.Policies.Repo, s.cfg.Project.Org, repo) { return local } - return "" + return s.remoteAgentsRepoRoot(repo) } // sameRepo reports whether a git source's repo field names org/repo. The source diff --git a/src/pkg/scheduler/kick_priming_test.go b/src/pkg/scheduler/kick_priming_test.go index 9a20f413d6..6a8b0121be 100644 --- a/src/pkg/scheduler/kick_priming_test.go +++ b/src/pkg/scheduler/kick_priming_test.go @@ -1,7 +1,9 @@ package scheduler import ( + "context" "encoding/json" + "errors" "log/slog" "os" "path/filepath" @@ -329,3 +331,120 @@ func TestInceptionVars_GreenfieldEmptyIdeaDoesNotFallBack(t *testing.T) { t.Errorf("idea = %q, want empty: RepoURL fallback is brownfield-only", idea) } } + +// --- remoteAgentsRepoRoot (GitHub fallback, #17090) --- + +type fakeAgentsMdFetcher struct { + body string + err error + calls int + last [4]string +} + +func (f *fakeAgentsMdFetcher) GetFileContentRef(_ context.Context, owner, repo, path, ref string) (string, error) { + f.calls++ + f.last = [4]string{owner, repo, path, ref} + return f.body, f.err +} + +func useTempRemoteAgentsMdDir(t *testing.T) { + t.Helper() + old := remoteAgentsMdDir + remoteAgentsMdDir = t.TempDir() + t.Cleanup(func() { remoteAgentsMdDir = old }) +} + +// With no local root and no fetcher wired the behaviour is unchanged: nothing. +func TestAgentsRepoRoot_NoFetcherStaysEmpty(t *testing.T) { + useTempRemoteAgentsMdDir(t) + cfg := &config.Config{} + cfg.Project.Org = "hivecommons" + s := New(cfg, slog.Default()) + if got := s.agentsRepoRoot("hive"); got != "" { + t.Errorf("agentsRepoRoot() = %q, want empty with no fetcher", got) + } +} + +// A hosted spoke with no checkout gets the primary repo's AGENTS.md from the +// forge's default branch, materialised so the existing priming path injects it, +// and one fetch serves every kick inside the TTL. +func TestAgentsRepoRoot_FetchesFromGitHubWhenNoLocalRoot(t *testing.T) { + useTempRemoteAgentsMdDir(t) + const body = "Always add a changelog.d fragment." + f := &fakeAgentsMdFetcher{body: body + "\n"} + + cfg := &config.Config{} + cfg.Project.Org = "hivecommons" + cfg.Project.PrimaryRepo = "hive" + s := New(cfg, slog.Default()) + s.SetAgentsMdFetcher(f) + + root := s.agentsRepoRoot("hive") + if want := filepath.Join(remoteAgentsMdDir, "hivecommons", "hive"); root != want { + t.Fatalf("agentsRepoRoot() = %q, want %q", root, want) + } + if f.last != [4]string{"hivecommons", "hive", "AGENTS.md", ""} { + t.Errorf("fetched %v, want owner/repo AGENTS.md at the default branch", f.last) + } + if got := s.primeAgentsMd(root); !strings.Contains(got, body) { + t.Fatalf("fetched AGENTS.md did not reach the injection; got %q", got) + } + + for i := 0; i < 3; i++ { + if again := s.agentsRepoRoot("hivecommons/hive"); again != root { + t.Fatalf("agentsRepoRoot() = %q on repeat, want cached %q", again, root) + } + } + if f.calls != 1 { + t.Errorf("fetch calls = %d, want 1 (served from cache inside the TTL)", f.calls) + } +} + +// A repo without AGENTS.md (404) yields no root and is not re-asked inside +// the TTL; a later failure keeps serving the last-known-good copy. +func TestAgentsRepoRoot_RemoteMissAndLastKnownGood(t *testing.T) { + useTempRemoteAgentsMdDir(t) + f := &fakeAgentsMdFetcher{err: errors.New("404 Not Found")} + cfg := &config.Config{} + cfg.Project.Org = "hivecommons" + s := New(cfg, slog.Default()) + s.SetAgentsMdFetcher(f) + + if got := s.agentsRepoRoot("hive"); got != "" { + t.Fatalf("agentsRepoRoot() = %q, want empty on 404", got) + } + s.agentsRepoRoot("hive") + if f.calls != 1 { + t.Errorf("fetch calls = %d, want 1 (negative result cached)", f.calls) + } + + // Expire the cache, succeed once, then fail: the good copy must survive. + s.remoteAgents = nil + f.err, f.body = nil, "rule\n" + root := s.agentsRepoRoot("hive") + if root == "" { + t.Fatal("agentsRepoRoot() empty after a successful fetch") + } + s.remoteAgents["hivecommons/hive"] = remoteAgentsMd{root: root, fetched: time.Now().Add(-2 * remoteAgentsMdTTL)} + f.err = errors.New("502 Bad Gateway") + if got := s.agentsRepoRoot("hive"); got != root { + t.Errorf("agentsRepoRoot() = %q after refresh failure, want last-known-good %q", got, root) + } +} + +// A configured local root still wins over the forge: no fetch happens. +func TestAgentsRepoRoot_LocalRootBeatsFetch(t *testing.T) { + useTempRemoteAgentsMdDir(t) + f := &fakeAgentsMdFetcher{body: "x"} + cfg := &config.Config{} + cfg.Project.Org = "hivecommons" + cfg.Project.CheckoutsDir = "/data/checkouts" + s := New(cfg, slog.Default()) + s.SetAgentsMdFetcher(f) + if got, want := s.agentsRepoRoot("hive"), filepath.Join("/data/checkouts", "hive"); got != want { + t.Errorf("agentsRepoRoot() = %q, want %q", got, want) + } + if f.calls != 0 { + t.Errorf("fetch calls = %d, want 0 when a local root exists", f.calls) + } +} diff --git a/src/pkg/scheduler/scheduler.go b/src/pkg/scheduler/scheduler.go index a9289fd3e9..639715b522 100644 --- a/src/pkg/scheduler/scheduler.go +++ b/src/pkg/scheduler/scheduler.go @@ -16,15 +16,20 @@ import ( ) type Scheduler struct { - surgeDuration func() (time.Duration, bool) - cfg *config.Config - primer *knowledge.Primer - inception *knowledge.InceptionEngine - lastActionable *github.ActionableResult - logger *slog.Logger - primerCapLogged sync.Map // agent|cap_bytes -> omitted fact count last logged at INFO - primedSlugs sync.Map // agent -> []string slugs the last kick's knowledge block rendered (#17052) - promptResolver *promptsrc.Resolver + surgeDuration func() (time.Duration, bool) + cfg *config.Config + primer *knowledge.Primer + inception *knowledge.InceptionEngine + lastActionable *github.ActionableResult + logger *slog.Logger + primerCapLogged sync.Map // agent|cap_bytes -> omitted fact count last logged at INFO + primedSlugs sync.Map // agent -> []string slugs the last kick's knowledge block rendered (#17052) + promptResolver *promptsrc.Resolver + agentsMdFetcher AgentsMdFetcher + // remoteAgents caches the materialised AGENTS.md root per "owner/repo" + // for the GitHub fallback in remoteAgentsRepoRoot (#17090). + remoteAgents map[string]remoteAgentsMd + remoteAgentsMu sync.Mutex auditFunc AuditFunc advisoryFunc AdvisoryFunc classifierUnavailableFunc func(error) diff --git a/src/pkg/scheduler/templates.go b/src/pkg/scheduler/templates.go index 769057bb09..50ccaf6784 100644 --- a/src/pkg/scheduler/templates.go +++ b/src/pkg/scheduler/templates.go @@ -365,7 +365,8 @@ func (s *Scheduler) substituteTemplateWithVars(template string, actionable *gith repoRoot := s.agentsRepoRoot(primaryRepo) // Additive: prepend the repo's AGENTS.md instructions + requested skills to - // the injected knowledge, when a local checkout root is available. This is a + // the injected knowledge, when a checkout root is available — local, or the + // GitHub-fetched copy a hosted spoke falls back to (#17090). This is a // guarded, single call point — it returns "" (and never errors) when no // AGENTS.md exists, so it is a no-op for repos that don't use the convention. //