diff --git a/changelog.d/fixed-knowledge-priming-query.md b/changelog.d/fixed-knowledge-priming-query.md new file mode 100644 index 0000000000..1d779e877c --- /dev/null +++ b/changelog.d/fixed-knowledge-priming-query.md @@ -0,0 +1 @@ +- **Primed knowledge is now about the task, not about the hive's own routing.** The "# Relevant Knowledge" search built its query from an issue's labels, complexity tier and title words. On hive-routed work those were mostly the hive's own: `epic`, `approved-direction`, `stage/010-actionable`, `agent/reviewer`, `medium`, and the `[reviewer] Review backlog (style):` title stem. They match review and planning notes in every repo, so a scanner task on `projectbluefin/utah-packages#879` was primed with entries about other repos, and on the projectbluefin hive the reviewer, quality and sec-check kicks carried the same section whatever their work. The query now leaves out what the hive itself put on the item: its routing label families (`stage/`, `agent/`, `origin/`, `hive/`, `hive:`, `hive-pause/`, `author/`, `triage/`, `size/`, `lifecycle/`, `do-not-merge`, `needs-`, `dco-signoff`, `sentinel-`), Prow's (`tide/`, `ok-to-test`, `release-note`, `cncf-cla:`), the labels `approved-direction`, `hive-review`, `lgtm`, `approved` and `claimed`, the hive's never-claim labels (`claims.never_claim_labels`, default `epic`), the complexity tier, a leading `[]` tag when it names a hive lane, origin source or configured agent (a person's `[nvidia] …` keeps `nvidia`), and the review backlog stem, cut down to its perspective (`style`, `security`). When that leaves nothing to search on, or what is left matches no entry, the search runs again with the query earlier releases built, so no kick loses the section it got before. This applies to resident kicks and to contributor role guidance alike. See [Knowledge: the kick primer](src/docs/agent-knowledge-and-memory.md#1-knowledge-the-kick-primer-push). diff --git a/src/docs/agent-knowledge-and-memory.md b/src/docs/agent-knowledge-and-memory.md index 3f6392b28d..723cf1faa5 100644 --- a/src/docs/agent-knowledge-and-memory.md +++ b/src/docs/agent-knowledge-and-memory.md @@ -27,8 +27,13 @@ memory. registerKnowledgeStores (cmd/hive/boot_knowledge_primer.go) ▼ ┌───────────────── Scheduler.primeKnowledge (pkg/scheduler/kick_priming.go) ───┐ - │ 1. extractKeywords ← titles + labels of the agent's first │ - │ maxIssuesToPrime (=5) queued issues │ + │ 1. extractKeywordsFor ← titles + labels of the agent's first │ + │ maxIssuesToPrime (=5) queued issues, minus what the hive put │ + │ there: routing labels (stage/, agent/, origin/, hive/, needs-, │ + │ never-claim, Prow's, …), the complexity tier, a filer tag │ + │ naming a hive agent and the "Review backlog (

):" stem; when │ + │ that leaves nothing, or matches no entry in step 3, the search │ + │ uses the pre-filter query instead (legacyKeywords) │ │ 2. scope ← knowledge.agent_scopes[] (or base agent for a replica) │ │ 3. Primer.PrimeScoped (pkg/knowledge/primer.go) │ │ search every layer/store → merge by layer precedence → │ diff --git a/src/pkg/scheduler/kick_priming.go b/src/pkg/scheduler/kick_priming.go index c5138dfdf7..541b5e3911 100644 --- a/src/pkg/scheduler/kick_priming.go +++ b/src/pkg/scheduler/kick_priming.go @@ -4,13 +4,19 @@ import ( "context" "fmt" "log/slog" + "regexp" + "slices" "strings" "github.com/hivecommons/hive/crux/agentsmd" "github.com/hivecommons/hive/crux/skillreg" + "github.com/hivecommons/hive/pkg/authorkind" + "github.com/hivecommons/hive/pkg/claims" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/issueshape" "github.com/hivecommons/hive/pkg/knowledge" + "github.com/hivecommons/hive/pkg/stagelabels" ) const maxIssuesToPrime = 5 @@ -380,7 +386,7 @@ func (s *Scheduler) renderKnowledge(primer *knowledge.Primer, cfg *config.Config return knowledgeInjection{}, nil } - keywords := extractKeywords(issues) + keywords := extractKeywordsFor(cfg, issues) if len(keywords) == 0 { s.logger.Debug("knowledge primer: no keywords extracted from issues", "issue_count", len(issues)) return knowledgeInjection{}, nil @@ -395,7 +401,19 @@ func (s *Scheduler) renderKnowledge(primer *knowledge.Primer, cfg *config.Config } scope = &agentScope } - primed := primer.PrimeForKick(context.Background(), scope, kickContextFor(cfg, agentName, issues), keywords) + kc := kickContextFor(cfg, agentName, issues) + primed := primer.PrimeForKick(context.Background(), scope, kc, keywords) + // The issue's own terms matched nothing: search again with the terms + // earlier releases used, so leaving the hive's routing words out never + // costs a kick the section it got before. + if len(primed.Facts) == 0 { + if legacy := legacyKeywords(issues); len(legacy) > 0 && !slices.Equal(legacy, keywords) { + s.logger.Info("knowledge primer: no entry matched the issue terms, retrying with the full query", + "sample", keywordSample(keywords), "full_sample", keywordSample(legacy)) + keywords = legacy + primed = primer.PrimeForKick(context.Background(), scope, kc, keywords) + } + } result, omitted := primed.Render() inj := knowledgeInjection{ Section: result, @@ -572,34 +590,75 @@ func kickContextFor(cfg *config.Config, agentName string, issues []github.Issue) return kc } -// extractKeywords pulls searchable terms from issue labels and titles. +// extractKeywords is extractKeywordsFor with no hive config: the built-in +// lanes and origin sources are the only filers, and the default never-claim +// labels apply. +func extractKeywords(issues []github.Issue) []string { + return extractKeywordsFor(nil, issues) +} + +// extractKeywordsFor pulls searchable terms from issue labels and titles. // Title words are included because labels alone are often all noise // (triage/accepted, kind/bug) and produce zero keywords after filtering. -func extractKeywords(issues []github.Issue) []string { +// +// What the hive itself puts on an item is dropped, because it says where the +// item sits in the hive, not what it is about: its routing labels +// (isRoutingLabel), its complexity tier, the "[]" filer tag on a title +// it filed, and the "Review backlog" stem. On utah-packages#879 those made up +// most of the query (stage/010-actionable, agent/reviewer, medium, "reviewer +// review backlog") and pulled in review notes from unrelated repos. When +// nothing is left, the terms earlier releases used are returned instead +// (legacyKeywords); renderKnowledge also falls back to them when these terms +// match no entry. +func extractKeywordsFor(cfg *config.Config, issues []github.Issue) []string { + filers := hiveFilers(cfg) + neverClaim := neverClaimLabels(cfg) + keywords := collectKeywords(issues, + func(label string) bool { return !isRoutingLabel(label) && !neverClaim[label] }, + false, + func(title string) string { return searchableTitle(filers, title) }) + if len(keywords) == 0 { + return legacyKeywords(issues) + } + return keywords +} + +// legacyKeywords is the query earlier releases built: every label outside +// noiseLabels, the complexity tier, and every title word. +func legacyKeywords(issues []github.Issue) []string { + return collectKeywords(issues, + func(label string) bool { return !isNoiseLabel(label) }, + true, + func(title string) string { return title }) +} + +// collectKeywords is the shared walk: each issue's kept labels, then its tier +// when withTier is set, then the words of title(issue.Title), lower-cased, +// noise words dropped and each term once. +func collectKeywords(issues []github.Issue, keepLabel func(string) bool, withTier bool, title func(string) string) []string { seen := make(map[string]bool) var keywords []string + add := func(word string) { + if !seen[word] { + keywords = append(keywords, word) + seen[word] = true + } + } for _, issue := range issues { for _, label := range issue.Labels { - lower := strings.ToLower(label) - if !seen[lower] && !isNoiseLabel(lower) { - keywords = append(keywords, lower) - seen[lower] = true + if lower := strings.ToLower(label); keepLabel(lower) { + add(lower) } } - if issue.ComplexityTier != "" { - tier := strings.ToLower(issue.ComplexityTier) - if !seen[tier] { - keywords = append(keywords, tier) - seen[tier] = true - } + if withTier && issue.ComplexityTier != "" { + add(strings.ToLower(issue.ComplexityTier)) } - for _, word := range splitTitleWords(issue.Title) { - if !seen[word] && !isNoiseWord(word) { - keywords = append(keywords, word) - seen[word] = true + for _, word := range splitTitleWords(title(issue.Title)) { + if !isNoiseWord(word) { + add(word) } } } @@ -607,6 +666,65 @@ func extractKeywords(issues []github.Issue) []string { return keywords } +// reviewBacklogTitleRE matches the stem the hive puts on its review backlog +// items, "[reviewer] Review backlog (): ", with or +// without the filer tag (the same shape as pkg/github's +// reviewBacklogTitleCategoryRE). +var reviewBacklogTitleRE = regexp.MustCompile(`^(?:\[[^\]\s]+\]\s*)?Review backlog \(([^)]+)\):\s*`) + +// searchableTitle is the part of an issue title that says what the issue is +// about. A review backlog title is cut down to its perspective, which names +// the kind of finding ("security", "style"), and the finding; the words +// "review backlog" match every review-related entry in the vault whatever the +// repo. Otherwise a leading "[]" is dropped only when it names a hive +// filer (hiveFilers): any lowercase word in brackets has the filer shape, and +// a person's "[nvidia] driver fails to build" keeps "nvidia". +func searchableTitle(filers map[string]bool, title string) string { + title = strings.TrimSpace(title) + if m := reviewBacklogTitleRE.FindStringSubmatch(title); m != nil { + return m[1] + " " + title[len(m[0]):] + } + if tag := issueshape.AgentPrefix(title); tag != "" && filers[strings.ToLower(tag)] { + return issueshape.StripAgentPrefix(title) + } + return title +} + +// hiveFilers is the set of names that file issues for the hive: the built-in +// lanes (scanner, reviewer, …), the other origin sources (selfreport, +// governor, …) and the agents configured on this hive. +func hiveFilers(cfg *config.Config) map[string]bool { + filers := make(map[string]bool) + for _, lane := range config.KnownOriginLanes { + filers[lane] = true + } + for _, source := range config.KnownOriginSources() { + filers[source] = true + } + if cfg != nil { + for name := range cfg.Agents { + filers[strings.ToLower(name)] = true + filers[strings.ToLower(cfg.BaseAgentName(name))] = true + } + } + return filers +} + +// neverClaimLabels is the hive's never-claim set (claims.never_claim_labels, +// default "epic"), lower-cased. Those labels mark trackers the hive will not +// work, so like routing labels they say nothing about the work. +func neverClaimLabels(cfg *config.Config) map[string]bool { + labels := claims.DefaultNeverClaimLabels + if cfg != nil { + labels = cfg.Governor.Claims.EffectiveNeverClaimLabels() + } + out := make(map[string]bool, len(labels)) + for _, l := range labels { + out[strings.ToLower(strings.TrimSpace(l))] = true + } + return out +} + // splitTitleWords extracts lowercase words from an issue title, dropping // short words and punctuation. func splitTitleWords(title string) []string { @@ -662,3 +780,45 @@ var noiseLabels = map[string]bool{ func isNoiseLabel(label string) bool { return noiseLabels[label] } + +// routingLabelPrefixes are the label families the hive, Prow and the review +// bots use to route and gate work: where an item sits in the pipeline +// (stage/, needs-), who holds or filed it (agent/ and origin/, see +// isRoutingLabel; hive/claimed-by-…; hive: churn-triaged), a hive's hold +// (hive-pause/), the author class (author/human, …), and review, test or +// merge state (tide/, do-not-merge, release-note, cncf-cla:). They say nothing +// about what the issue is about, and they are on most items, so as search +// terms they match the same review and scanning entries for every task. +var routingLabelPrefixes = []string{ + stagelabels.Prefix, "hive/", "hive:", github.HiveHoldLabelPrefix, authorkind.Prefix, + "triage/", "size/", "lifecycle/", "do-not-merge", "needs-", "dco-signoff", + "sentinel-", "tide/", "release-note", "cncf-cla:", +} + +// routingLabels are single hive, Prow or review-bot labels outside those +// families. +var routingLabels = map[string]bool{ + "approved-direction": true, + "hive-review": true, + "lgtm": true, + "approved": true, + "claimed": true, + "ok-to-test": true, +} + +// isRoutingLabel reports whether a lower-cased label is one the hive's search +// leaves out: noiseLabels, and every routing label above. +func isRoutingLabel(label string) bool { + if noiseLabels[label] || routingLabels[label] { + return true + } + if config.IsAgentLaneLabel(label) || config.IsOriginLabel(label) { + return true + } + for _, prefix := range routingLabelPrefixes { + if strings.HasPrefix(label, prefix) { + return true + } + } + return false +} diff --git a/src/pkg/scheduler/kick_priming_query_test.go b/src/pkg/scheduler/kick_priming_query_test.go new file mode 100644 index 0000000000..d6fa09b950 --- /dev/null +++ b/src/pkg/scheduler/kick_priming_query_test.go @@ -0,0 +1,153 @@ +package scheduler + +import ( + "encoding/json" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "reflect" + "strings" + "sync" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/knowledge" +) + +// utah-packages#879 as a scanner task: before, the query was the hive's own +// routing labels, its complexity tier and the review backlog stem +// ("stage/010-actionable agent/reviewer kind/cleanup approved-direction +// medium reviewer review backlog style …"), so review notes from other repos +// outranked anything about utah-packages. The query keeps what the issue is +// about. +func TestExtractKeywords_DropsHiveRouting(t *testing.T) { + issue := github.Issue{ + Repo: "projectbluefin/utah-packages", + Number: 879, + Title: "[reviewer] Review backlog (style): spec file mixes tabs and spaces", + ComplexityTier: "Medium", + Labels: []string{"stage/010-actionable", "agent/reviewer", "kind/cleanup", "approved-direction", + "hive/claimed-by-scanner", "needs-triage", "epic", "claimed", "author/human", "hive-pause/h1", + "origin/governor", "ok-to-test", "tide/merge-method-squash", "do-not-merge"}, + } + got := extractKeywords([]github.Issue{issue}) + want := []string{"kind/cleanup", "style", "spec", "file", "mixes", "tabs", "spaces"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("keywords = %q\nwant %q", got, want) + } +} + +// An issue whose every term is hive routing gets the query earlier releases +// built, so the filter never leaves it with nothing to search on. +func TestExtractKeywords_AllRoutingFallsBackToLegacyQuery(t *testing.T) { + issue := github.Issue{ + Repo: "projectbluefin/utah", + Number: 12, + Title: "[ci-maintainer] Fix CI", + Labels: []string{"agent/ci-maintainer", "stage/010-actionable", "kind/bug"}, + } + got := extractKeywords([]github.Issue{issue}) + want := []string{"agent/ci-maintainer", "stage/010-actionable", "ci-maintainer"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("keywords = %q\nwant the legacy query %q", got, want) + } +} + +// A hive that configures its own never-claim labels gets those left out too, +// and the default "epic" then counts as an ordinary word. +func TestExtractKeywords_ConfiguredNeverClaimLabels(t *testing.T) { + cfg := &config.Config{} + cfg.Governor.Claims.NeverClaimLabels = []string{"Tracker"} + issue := github.Issue{Repo: "org/repo", Number: 3, Title: "Sidebar anchors drift", Labels: []string{"tracker", "epic"}} + got := extractKeywordsFor(cfg, []github.Issue{issue}) + want := []string{"epic", "sidebar", "anchors", "drift"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("keywords = %q, want %q", got, want) + } +} + +func TestSearchableTitle(t *testing.T) { + cfg := &config.Config{} + cfg.Agents = map[string]config.AgentConfig{"packager": {}} + filers := hiveFilers(cfg) + for title, want := range map[string]string{ + "[reviewer] Review backlog (security): token in log line": "security token in log line", + "[someone] Review backlog (style): long line": "style long line", + "Review backlog (style): long line": "style long line", + "[scanner] Sidebar anchors drift": "Sidebar anchors drift", + "[selfreport] Agent pane froze": "Agent pane froze", + "[packager] Bump the spec": "Bump the spec", + "[nvidia] driver fails to build on F44": "[nvidia] driver fails to build on F44", + "[ACMM L0] Add a CONTRIBUTING file": "[ACMM L0] Add a CONTRIBUTING file", + "Review the backlog of stale PRs": "Review the backlog of stale PRs", + } { + if got := searchableTitle(filers, title); got != want { + t.Errorf("searchableTitle(%q) = %q, want %q", title, got, want) + } + } + if got := searchableTitle(hiveFilers(nil), "[packager] Bump the spec"); got != "[packager] Bump the spec" { + t.Errorf("an unconfigured agent name was stripped as a filer: %q", got) + } +} + +func TestIsRoutingLabel(t *testing.T) { + for _, l := range []string{"kind/bug", "triage/accepted", "stage/010-actionable", "stage/stalled", "agent/reviewer", + "hive/claimed-by-scanner", "hive: churn-triaged", "origin/governor", "triage/needs-information", "size/l", + "needs-human", "do-not-merge", "do-not-merge/hold", "dco-signoff: yes", "sentinel-alert", "approved-direction", + "hive-review", "lgtm", "approved", "claimed", "author/human", "hive-pause/h1", "ok-to-test", "needs-ok-to-test", + "tide/merge-method-squash", "release-note-none", "cncf-cla: yes"} { + if !isRoutingLabel(l) { + t.Errorf("%q is a routing label and should not be a search term", l) + } + } + for _, l := range []string{"kind/cleanup", "priority/high", "area/packaging", "rpm", "stagehand", "agentic", "authoring", "epic"} { + if isRoutingLabel(l) { + t.Errorf("%q describes the work and should stay a search term", l) + } + } +} + +// The issue's own terms match nothing in the wiki, but the terms earlier +// releases used do: the kick still gets the section it got before, from a +// second search with the full query. +func TestPrimeKnowledge_NoHitFallsBackToLegacyQuery(t *testing.T) { + var ( + mu sync.Mutex + queries []string + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("q") + mu.Lock() + queries = append(queries, q) + mu.Unlock() + results := []map[string]interface{}{} + if strings.Contains(q, "ci-maintainer") { + results = append(results, map[string]interface{}{ + "slug": "ci-lane-notes", "title": "CI lane notes", "score": 0.9, + "type": "gotcha", "confidence": 0.9, "snippet": "LEGACY-HIT: rerun lint once before repairing.", + }) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{"total": len(results), "results": results}) + })) + defer srv.Close() + + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})) + s := newScheduler() + s.SetPrimer(knowledge.NewPrimer([]knowledge.LayerConfig{{Type: knowledge.LayerProject, URL: srv.URL, Shared: true}}, + knowledge.PrimerConfig{MaxFacts: 25}, logger)) + issue := makeIssue("projectbluefin/utah", 12, "[ci-maintainer] Flaky lint job", "", 5, + []string{"agent/ci-maintainer", "stage/010-actionable"}, false) + + got := s.primeKnowledge("ci-maintainer", []github.Issue{issue}) + mu.Lock() + defer mu.Unlock() + if !strings.Contains(got, "LEGACY-HIT") { + t.Fatalf("the kick lost the section the full query finds:\n%s\nqueries: %q", got, queries) + } + if len(queries) != 2 || queries[0] != "flaky lint job" || !strings.Contains(queries[1], "agent/ci-maintainer") { + t.Fatalf("queries = %q, want the issue's terms first and the full query second", queries) + } +}