diff --git a/changelog.d/added-knowledge-seed-hosted-spokes.md b/changelog.d/added-knowledge-seed-hosted-spokes.md new file mode 100644 index 0000000000..f504cdf48f --- /dev/null +++ b/changelog.d/added-knowledge-seed-hosted-spokes.md @@ -0,0 +1 @@ +- Hives with no `knowledge.git_sources` now index their primary repo's knowledge dir (`bead_synthesizer.publish.path`, else `docs/knowledge/` or `src/docs/knowledge/`) as a seeded `-knowledge` project-layer git source, and `knowledge.primer.max_bytes` defaults to 8192 instead of 4096. diff --git a/src/cmd/hive/boot_knowledge_seed.go b/src/cmd/hive/boot_knowledge_seed.go new file mode 100644 index 0000000000..79cf20ea94 --- /dev/null +++ b/src/cmd/hive/boot_knowledge_seed.go @@ -0,0 +1,114 @@ +package main + +import ( + "context" + "log/slog" + "strings" + + "github.com/hivecommons/hive/pkg/config" +) + +// knowledgeSeedProbe is the slice of *github.Client the git-source seed +// needs: the primary repo's default branch and whether a path exists there. +type knowledgeSeedProbe interface { + DefaultBranch(ctx context.Context, owner, repo string) (string, error) + PathExistsAtRef(ctx context.Context, owner, repo, path, ref string) (bool, error) +} + +// knowledgeSeedCandidatePaths are probed in order when +// knowledge.bead_synthesizer.publish.path is unset; the first that exists +// in the primary repo becomes the seeded git source's subpath. +var knowledgeSeedCandidatePaths = []string{"docs/knowledge", "src/docs/knowledge"} + +// knowledgeSeedLayer is the layer a repo's own knowledge dir is indexed at. +const knowledgeSeedLayer = "project" + +// seedKnowledgeGitSource gives a hive with no knowledge.git_sources one +// entry pointing at its primary repo's knowledge directory, so hosted spokes +// index the repo's curated entries without per-spoke hand config. It runs +// in memory at every boot and never writes hive.yaml: an operator list (even +// an explicit empty `git_sources: []`, which is what a dashboard disconnect +// saves) is left alone, so the decision is idempotent and never overwrites +// configuration. It reports whether an entry was appended. +func seedKnowledgeGitSource(ctx context.Context, cfg *config.Config, probe knowledgeSeedProbe, logger *slog.Logger) bool { + if cfg == nil { + return false + } + if cfg.Knowledge.GitSources != nil { + logger.Debug("knowledge: git source seed skipped", "reason", "knowledge.git_sources is operator-set", "count", len(cfg.Knowledge.GitSources)) + return false + } + repo := strings.TrimSpace(cfg.Project.PrimaryRepo) + if repo == "" && len(cfg.Project.Repos) > 0 { + repo = strings.TrimSpace(cfg.Project.Repos[0]) + } + owner, name, ok := strings.Cut(repo, "/") + if !ok || owner == "" || name == "" { + logger.Debug("knowledge: git source seed skipped", "reason", "no primary repo") + return false + } + if cfg.Project.ForgeKind() != config.ForgeGitHub { + logger.Debug("knowledge: git source seed skipped", "reason", "forge is not github", "forge", cfg.Project.ForgeKind()) + return false + } + if probe == nil { + logger.Debug("knowledge: git source seed skipped", "reason", "no github client", "repo", repo) + return false + } + branch, err := probe.DefaultBranch(ctx, owner, name) + if err != nil { + logger.Debug("knowledge: git source seed skipped", "reason", "default branch lookup failed", "repo", repo, "error", err) + return false + } + var subpath string + for _, candidate := range knowledgeSeedPaths(cfg.Knowledge.BeadSynthesizer.Publish.Path) { + exists, err := probe.PathExistsAtRef(ctx, owner, name, candidate, branch) + if err != nil { + logger.Debug("knowledge: git source seed skipped", "reason", "path probe failed", "repo", repo, "path", candidate, "error", err) + return false + } + if exists { + subpath = candidate + break + } + } + if subpath == "" { + logger.Debug("knowledge: git source seed skipped", "reason", "no knowledge dir in primary repo", "repo", repo, "branch", branch) + return false + } + base := strings.TrimRight(cfg.GitHub.BaseURL, "/") + if base == "" { + base = config.DefaultGitHubBaseURL + } + gs := config.GitSourceConfigYAML{ + Name: name + "-knowledge", + URL: base + "/" + repo, + Branch: branch, + Subpath: subpath, + Layer: knowledgeSeedLayer, + } + cfg.Knowledge.GitSources = append(cfg.Knowledge.GitSources, gs) + logger.Info("knowledge: seeded git source "+gs.Name+" from "+repo+"/"+subpath, "branch", branch, "layer", gs.Layer) + return true +} + +// knowledgeSeedPaths orders the subpaths to probe: the configured bead-synth +// publish path first (that is where this repo keeps its knowledge dir), then +// the conventional locations, without repeats. +func knowledgeSeedPaths(publishPath string) []string { + var out []string + seen := map[string]bool{} + add := func(p string) { + p = strings.Trim(strings.TrimSpace(p), "/") + if p == "" || seen[p] { + return + } + seen[p] = true + out = append(out, p) + } + add(publishPath) + for _, p := range knowledgeSeedCandidatePaths { + add(p) + } + return out +} diff --git a/src/cmd/hive/boot_knowledge_seed_test.go b/src/cmd/hive/boot_knowledge_seed_test.go new file mode 100644 index 0000000000..dd75b8c893 --- /dev/null +++ b/src/cmd/hive/boot_knowledge_seed_test.go @@ -0,0 +1,197 @@ +package main + +import ( + "context" + "errors" + "log/slog" + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +// fakeSeedProbe answers the seed's two GitHub questions from fixed data and +// records the paths it was asked about. +type fakeSeedProbe struct { + branch string + branchErr error + exists map[string]bool + probeErr error + asked []string +} + +func (f *fakeSeedProbe) DefaultBranch(_ context.Context, _, _ string) (string, error) { + return f.branch, f.branchErr +} + +func (f *fakeSeedProbe) PathExistsAtRef(_ context.Context, _, _, path, _ string) (bool, error) { + f.asked = append(f.asked, path) + if f.probeErr != nil { + return false, f.probeErr + } + return f.exists[path], nil +} + +func TestSeedKnowledgeGitSource(t *testing.T) { + operatorSet := []config.GitSourceConfigYAML{{Name: "mine", URL: "https://github.com/acme/wiki", Layer: "org"}} + tests := []struct { + name string + cfg func() *config.Config + probe *fakeSeedProbe + nilProbe bool + want *config.GitSourceConfigYAML + wantAsked []string + wantLen int + }{ + { + name: "primary repo with docs/knowledge", + cfg: func() *config.Config { return seedCfg("acme/widgets") }, + probe: &fakeSeedProbe{branch: "main", exists: map[string]bool{"docs/knowledge": true}}, + want: &config.GitSourceConfigYAML{Name: "widgets-knowledge", URL: "https://github.com/acme/widgets", Branch: "main", Subpath: "docs/knowledge", Layer: "project"}, + wantAsked: []string{"docs/knowledge"}, + wantLen: 1, + }, + { + name: "falls through to src/docs/knowledge on the default branch", + cfg: func() *config.Config { return seedCfg("hivecommons/hive") }, + probe: &fakeSeedProbe{branch: "v5", exists: map[string]bool{"src/docs/knowledge": true}}, + want: &config.GitSourceConfigYAML{Name: "hive-knowledge", URL: "https://github.com/hivecommons/hive", Branch: "v5", Subpath: "src/docs/knowledge", Layer: "project"}, + wantAsked: []string{"docs/knowledge", "src/docs/knowledge"}, + wantLen: 1, + }, + { + name: "bead-synth publish path is probed first and wins", + cfg: func() *config.Config { + c := seedCfg("acme/widgets") + c.Knowledge.BeadSynthesizer.Publish.Path = "knowledge/facts/" + return c + }, + probe: &fakeSeedProbe{branch: "main", exists: map[string]bool{"knowledge/facts": true, "docs/knowledge": true}}, + want: &config.GitSourceConfigYAML{Name: "widgets-knowledge", URL: "https://github.com/acme/widgets", Branch: "main", Subpath: "knowledge/facts", Layer: "project"}, + wantAsked: []string{"knowledge/facts"}, + wantLen: 1, + }, + { + name: "publish path equal to a default is not probed twice", + cfg: func() *config.Config { + c := seedCfg("acme/widgets") + c.Knowledge.BeadSynthesizer.Publish.Path = "docs/knowledge/" + return c + }, + probe: &fakeSeedProbe{branch: "main"}, + wantAsked: []string{"docs/knowledge", "src/docs/knowledge"}, + }, + { + name: "GHES base URL is used for the clone URL", + cfg: func() *config.Config { + c := seedCfg("acme/widgets") + c.GitHub.BaseURL = "https://github.example.com/" + return c + }, + probe: &fakeSeedProbe{branch: "main", exists: map[string]bool{"docs/knowledge": true}}, + want: &config.GitSourceConfigYAML{Name: "widgets-knowledge", URL: "https://github.example.com/acme/widgets", Branch: "main", Subpath: "docs/knowledge", Layer: "project"}, + wantAsked: []string{"docs/knowledge"}, + wantLen: 1, + }, + { + name: "no knowledge dir anywhere skips", + cfg: func() *config.Config { return seedCfg("acme/widgets") }, + probe: &fakeSeedProbe{branch: "main"}, + wantAsked: []string{"docs/knowledge", "src/docs/knowledge"}, + }, + { + name: "operator list is never touched", + cfg: func() *config.Config { + c := seedCfg("acme/widgets") + c.Knowledge.GitSources = append([]config.GitSourceConfigYAML(nil), operatorSet...) + return c + }, + probe: &fakeSeedProbe{branch: "main", exists: map[string]bool{"docs/knowledge": true}}, + wantLen: 1, + }, + { + name: "explicit empty list opts out", + cfg: func() *config.Config { + c := seedCfg("acme/widgets") + c.Knowledge.GitSources = []config.GitSourceConfigYAML{} + return c + }, + probe: &fakeSeedProbe{branch: "main", exists: map[string]bool{"docs/knowledge": true}}, + }, + { + name: "falls back to the first project repo", + cfg: func() *config.Config { + c := seedCfg("") + c.Project.Repos = []string{"acme/first", "acme/second"} + return c + }, + probe: &fakeSeedProbe{branch: "main", exists: map[string]bool{"docs/knowledge": true}}, + want: &config.GitSourceConfigYAML{Name: "first-knowledge", URL: "https://github.com/acme/first", Branch: "main", Subpath: "docs/knowledge", Layer: "project"}, + wantAsked: []string{"docs/knowledge"}, + wantLen: 1, + }, + { + name: "no primary repo skips", + cfg: func() *config.Config { return seedCfg("") }, + probe: &fakeSeedProbe{branch: "main", exists: map[string]bool{"docs/knowledge": true}}, + }, + { + name: "non-github forge skips", + cfg: func() *config.Config { + c := seedCfg("acme/widgets") + c.Project.Forge = config.ForgeGitLab + return c + }, + probe: &fakeSeedProbe{branch: "main", exists: map[string]bool{"docs/knowledge": true}}, + }, + { + name: "no github client skips", + cfg: func() *config.Config { return seedCfg("acme/widgets") }, + nilProbe: true, + }, + { + name: "default branch lookup failure skips", + cfg: func() *config.Config { return seedCfg("acme/widgets") }, + probe: &fakeSeedProbe{branchErr: errors.New("boom"), exists: map[string]bool{"docs/knowledge": true}}, + }, + { + name: "non-404 probe error skips rather than guessing", + cfg: func() *config.Config { return seedCfg("acme/widgets") }, + probe: &fakeSeedProbe{branch: "main", probeErr: errors.New("503")}, + wantAsked: []string{"docs/knowledge"}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := tc.cfg() + var probe knowledgeSeedProbe + if !tc.nilProbe { + probe = tc.probe + } + seeded := seedKnowledgeGitSource(context.Background(), cfg, probe, slog.Default()) + if seeded != (tc.want != nil) { + t.Fatalf("seeded = %v, want %v", seeded, tc.want != nil) + } + if len(cfg.Knowledge.GitSources) != tc.wantLen { + t.Fatalf("git_sources = %+v, want %d entries", cfg.Knowledge.GitSources, tc.wantLen) + } + if tc.want != nil && cfg.Knowledge.GitSources[0] != *tc.want { + t.Errorf("seeded = %+v, want %+v", cfg.Knowledge.GitSources[0], *tc.want) + } + if tc.probe != nil && !equalStrings(tc.probe.asked, tc.wantAsked) { + t.Errorf("probed paths = %v, want %v", tc.probe.asked, tc.wantAsked) + } + // Idempotent: a second pass sees the seeded (or operator) list and + // never appends again. + before := len(cfg.Knowledge.GitSources) + if seedKnowledgeGitSource(context.Background(), cfg, probe, slog.Default()) || len(cfg.Knowledge.GitSources) != before { + t.Errorf("second pass changed git_sources: %+v", cfg.Knowledge.GitSources) + } + }) + } +} + +func seedCfg(primary string) *config.Config { + c := &config.Config{} + c.Project.PrimaryRepo = primary + return c +} diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index f9155ad016..6192cb9168 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -2996,6 +2996,15 @@ func (b *boot) bootKnowledgeWith(deps bootKnowledgeDeps) { } } + // A hive with no git_sources gets its primary repo's knowledge dir + // (boot_knowledge_seed.go); bootGitHub has already run, so the probe can + // use the hive's own client. + var seedProbe knowledgeSeedProbe + if b.ghClient != nil { + seedProbe = b.ghClient + } + seedKnowledgeGitSource(b.ctx, b.cfg, seedProbe, b.logger) + // Auto-connect configured git sources (remote repos indexed as knowledge) for _, gsc := range b.cfg.Knowledge.GitSources { if b.knowledgeAPI == nil { diff --git a/src/docs/agent-knowledge-and-memory.md b/src/docs/agent-knowledge-and-memory.md index eddf4494d8..09aa7aa40b 100644 --- a/src/docs/agent-knowledge-and-memory.md +++ b/src/docs/agent-knowledge-and-memory.md @@ -36,7 +36,7 @@ memory. │ Fisher-Rao rerank → type priority → cap at max_facts (25) → │ │ expandWithGraph (never past the scope) → Context7 hints │ │ 4. PrimedKnowledge.Render (primer_render.go): "# Relevant Knowledge", │ - │ grouped by type, whole facts dropped past max_bytes (4096), │ + │ grouped by type, whole facts dropped past max_bytes (8192), │ │ "_… truncated: N facts / M bytes omitted (size cap)._" marker │ └──────────────────────────────────────────────────────────────────────────────┘ │ @@ -55,7 +55,17 @@ configured `knowledge.vaults` (personal layer), `knowledge.git_sources` organic source on a running hive), plus any HTTP wiki layers in `knowledge.layers`. The repo's own docs are *not* a source unless you index them through a git source, connector or code map -([knowledge-curator.md](knowledge-curator.md)). +([knowledge-curator.md](knowledge-curator.md)) — with one seeded default: +a hive whose `knowledge.git_sources` key is absent gets one entry for its +primary repo's knowledge directory at boot +(`seedKnowledgeGitSource`, `src/cmd/hive/boot_knowledge_seed.go`). The +probe asks GitHub for the repo's default branch, then for +`knowledge.bead_synthesizer.publish.path` if set, else `docs/knowledge/` +then `src/docs/knowledge/`; the first that exists becomes +`-knowledge` at the `project` layer. Nothing is written to +`hive.yaml`; an operator list — including an explicit `git_sources: []`, +which is what a dashboard disconnect saves — is never touched, and a +missing dir, non-GitHub forge or absent client skips with a DEBUG line. **How it is scoped.** @@ -86,7 +96,7 @@ them through a git source, connector or code map | Block | Cap | Config | |---|---|---| -| `# Relevant Knowledge` | 25 facts, 4096 bytes (floor 512) | `knowledge.primer.max_facts`, `knowledge.primer.max_bytes` | +| `# Relevant Knowledge` | 25 facts, 8192 bytes (floor 512) | `knowledge.primer.max_facts`, `knowledge.primer.max_bytes` | | Repo `AGENTS.md` | 8192 bytes (floor 2048), whole sections | `project.agents_md_max_bytes` (`agentsmd.DefaultMaxInjectionBytes`) | | Declared skills | 8192 bytes, whole skills | `maxSkillsInjectionBytes` in `kick_priming.go` (fixed) | diff --git a/src/pkg/config/knowledge_config.go b/src/pkg/config/knowledge_config.go index 34597e07c5..da88db60f0 100644 --- a/src/pkg/config/knowledge_config.go +++ b/src/pkg/config/knowledge_config.go @@ -128,7 +128,7 @@ const ( DefaultCuratorSchedule = "daily" DefaultCuratorAutoPromoteThreshold = 0.9 DefaultKnowledgePrimerMaxFacts = 25 - DefaultKnowledgePrimerMaxBytes = 4096 + DefaultKnowledgePrimerMaxBytes = 8192 DefaultKnowledgePrimerMergeStrategy = "precedence" DefaultKnowledgeAdvisorMaxPerKick = 3 ) @@ -313,7 +313,7 @@ type KnowledgePrimer struct { MergeStrategy string `yaml:"merge_strategy"` // MaxBytes caps the rendered knowledge primer per kick; facts past it are // dropped whole and a marker says so. Zero takes the knowledge package - // default (4096). + // default (8192). MaxBytes int `yaml:"max_bytes"` } diff --git a/src/pkg/dashboard/knowledge_settings_parity_test.go b/src/pkg/dashboard/knowledge_settings_parity_test.go index 95d26b3bd2..f7114d0968 100644 --- a/src/pkg/dashboard/knowledge_settings_parity_test.go +++ b/src/pkg/dashboard/knowledge_settings_parity_test.go @@ -25,7 +25,7 @@ func TestKnowledgePipelineParityBatch8RoundTrip(t *testing.T) { MaxFactsPerCycle: 20, MaxAPICallsPerCycle: 40, VaultPath: "/data/vaults/bead-synth-wiki", } k.Curator = config.KnowledgeCurator{AutoPromoteThreshold: 0.9} - k.Primer = config.KnowledgePrimer{MaxFacts: 25, MaxBytes: 4096, MergeStrategy: "precedence"} + k.Primer = config.KnowledgePrimer{MaxFacts: 25, MaxBytes: 8192, MergeStrategy: "precedence"} before := *k for _, bad := range []map[string]any{ @@ -190,7 +190,7 @@ func TestKnowledgePipelineParityBatch8RoundTrip(t *testing.T) { t.Fatal(err) } if resp.Engine != "llm-wiki" || resp.BeadSynthesizer["vault_path"] != "/data/vaults/bead-synth-wiki" || - resp.Primer["merge_strategy"] != "precedence" || resp.Defaults["primer_max_bytes"] != float64(4096) { + resp.Primer["merge_strategy"] != "precedence" || resp.Defaults["primer_max_bytes"] != float64(config.DefaultKnowledgePrimerMaxBytes) { t.Fatalf("GET response: %s", rec.Body.String()) } } diff --git a/src/pkg/knowledge/primer_render.go b/src/pkg/knowledge/primer_render.go index 64f0000af0..f564c64d42 100644 --- a/src/pkg/knowledge/primer_render.go +++ b/src/pkg/knowledge/primer_render.go @@ -11,10 +11,11 @@ import ( // so. Override with knowledge.primer.max_bytes. const ( // DefaultPrimerMaxBytes is the rendered-prompt cap when - // PrimerConfig.MaxBytes is zero. 4 KiB is the "Top 5 safe wins" figure from - // the 2026-10 token-usage review (#12124): enough for a dozen typical facts, - // small next to the issue/PR lists it shares the kick with. - DefaultPrimerMaxBytes = 4096 + // PrimerConfig.MaxBytes is zero. The 2026-10 token-usage review (#12124) + // set 4 KiB; on hive's own spoke that dropped 15 of 25 primed facts per + // kick once the knowledge dir was indexed, so 8 KiB keeps most of the + // max_facts budget while staying small next to the issue/PR lists. + DefaultPrimerMaxBytes = 8192 // primerMinBytes is the floor a configured cap is raised to, so a typo // cannot produce a primer that is nothing but its own marker. primerMinBytes = 512