diff --git a/changelog.d/added-19046-capture-reviewer-rules.md b/changelog.d/added-19046-capture-reviewer-rules.md new file mode 100644 index 0000000000..e26c20ea97 --- /dev/null +++ b/changelog.d/added-19046-capture-reviewer-rules.md @@ -0,0 +1 @@ +- Rules and sources of truth that people state in review feedback on the hive's own agent PRs are now captured deterministically into the hive's knowledge, so they hold for future work and not only for the PR they were written on ([#19046](https://github.com/hivecommons/hive/issues/19046)). The review-feedback sweep classifies each comment in a burst from explicit phrase signals (for example "source of truth", "going forward", "capture this", "assessment criteria"), writes a rule to a local `review-rules` knowledge channel with its provenance (PR, comment, reviewer, association, time), deduplicates by comment id and by near-identical text, and replies on the PR "recorded as rule R-n in the hive's knowledge, applies to ..." so the reviewer can object. Only `OWNER`, `MEMBER` and `COLLABORATOR` reviewers are captured at once (`HIVE_REVIEW_FEEDBACK_RULE_TRUSTED_ASSOCIATIONS`); other reviewers' rules are queued as drafts for a person to approve. The kick primer now carries pinned entries whose scope admits the kick whatever its keywords are, so the quality agent receives its rules on its next run. The kicked agent may end its reply with a validated `hive:review-rules` block to sharpen a rule, and a rule whose proper home is another project's repo gets exactly one needs-human note with the proposed text instead of a write there. Retract with `hivectl knowledge state rule-r-1 deprecated`. Set `HIVE_REVIEW_FEEDBACK_RULES=off` to stop capture. diff --git a/src/cmd/hive/boot_knowledge_primer.go b/src/cmd/hive/boot_knowledge_primer.go index c8434fc5db..c4dbfd1b4e 100644 --- a/src/cmd/hive/boot_knowledge_primer.go +++ b/src/cmd/hive/boot_knowledge_primer.go @@ -2,10 +2,13 @@ package main import ( "fmt" + "os" + "path/filepath" "strings" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/knowledge" + "github.com/hivecommons/hive/pkg/reviewrules" ) // beadSynthVaultPath is where the bead synthesizer writes its facts. @@ -52,12 +55,48 @@ func (b *boot) registerKnowledgeStores(p *knowledge.Primer) { b.logger.Info("bead-synth vault registered with primer", "layer", layer) } } + b.registerReviewRuleVault(api, p) if gs := api.GraphStore(); gs != nil { p.SetGraphStore(gs) api.WireContext7Suggester(p) } } +// registerReviewRuleVault connects the local channel the review-feedback sweep +// writes reviewer-stated rules into (hivecommons/hive#19046) and registers it +// with p as a pinned store: an approved rule whose scope admits a kick reaches +// that kick whatever its keywords are. The channel shows in the dashboard's +// knowledge list like any other, with each rule's source comment and reviewer. +func (b *boot) registerReviewRuleVault(api *knowledge.KnowledgeAPI, p *knowledge.Primer) { + if !reviewrules.Enabled() { + return + } + dir := reviewrules.VaultDir() + store := api.GetVaultStore(dir) + if store == nil { + // Create the channel only beside the hive's other vaults. A host + // with no vaults directory gets the channel on the first capture + // and connects it at the next boot. + if _, err := os.Stat(filepath.Dir(dir)); err != nil { + if _, derr := os.Stat(dir); derr != nil { + return + } + } + if err := os.MkdirAll(dir, 0o755); err != nil { + b.logger.Debug("review-rule vault not created; reviewer rules are not primed", "dir", dir, "error", err) + return + } + if err := api.ConnectVault(dir, reviewrules.VaultName); err != nil { + b.logger.Warn("review-rule vault not connected; reviewer rules are not primed", "dir", dir, "error", err) + return + } + store = api.GetVaultStore(dir) + } + if store != nil { + p.AddPinnedFileStore(reviewrules.VaultName, store, knowledge.LayerProject) + } +} + // knowledgeRestartReason names what a live-registered primer cannot apply // because knowledgeAPI is the file-only fallback bootKnowledge builds when // knowledge.enabled is false at boot. Empty when nothing waits on a restart. diff --git a/src/cmd/hive/review_feedback_rules_wire_test.go b/src/cmd/hive/review_feedback_rules_wire_test.go new file mode 100644 index 0000000000..545da0c7e2 --- /dev/null +++ b/src/cmd/hive/review_feedback_rules_wire_test.go @@ -0,0 +1,354 @@ +package main + +import ( + "context" + "errors" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/knowledge" + "github.com/hivecommons/hive/pkg/reviewfeedback" + "github.com/hivecommons/hive/pkg/reviewrules" +) + +// The maintainer's comment on the techdocs analysis PR (hivecommons/hive#19046). +const maintainersFixture = ".project/maintainers.yaml is the source of truth for ownership and maintainership; capture this in the assessment knowledge base" + +const maintainersRuleText = ".project/maintainers.yaml is the source of truth for ownership and maintainership" + +const fixtureCommentURL = "https://github.com/cncf/techdocs/pull/424#discussion_r1001" + +type ruleHarness struct { + rules *reviewrules.Store + vault string + src *fakeFeedbackSource + kick *recordingKicker + now time.Time +} + +func newRuleHarness(t *testing.T, cfg reviewrules.Config) *ruleHarness { + t.Helper() + vault := filepath.Join(t.TempDir(), "review-rules") + rs, err := reviewrules.Open(t.TempDir(), vault, cfg) + if err != nil { + t.Fatal(err) + } + return &ruleHarness{rules: rs, vault: vault, src: &fakeFeedbackSource{byKey: map[string]github.PRFeedback{}}, kick: &recordingKicker{}, now: time.Now()} +} + +func (h *ruleHarness) sweep(t *testing.T) reviewFeedbackSweep { + t.Helper() + s := feedbackSweepFor(t, h.src, h.kick, routeTo("quality"), h.now) + s.Rules = h.rules + s.Lane = func(owner string) string { return owner } + s.Managed = func(repo string) bool { return repo == "hivecommons/hive" } + return s +} + +func (h *ruleHarness) feedback(items ...github.FeedbackItem) { + fb := openDraftFeedback() + fb.Items = items + h.src.byKey[feedbackTestKey] = fb +} + +func ruleItem(id, assoc, body string) github.FeedbackItem { + return github.FeedbackItem{ID: id, ThreadID: "T-" + id, Kind: github.PRCommentInline, Author: "nate-double-u", Body: body, Path: "analysis.md", Line: 3, + URL: "https://github.com/cncf/techdocs/pull/424#discussion_r" + strings.TrimPrefix(id, "inline:"), CreatedAt: time.Now().Add(-time.Hour), AuthorAssociation: assoc} +} + +func (h *ruleHarness) retrievedFor(t *testing.T, lane string) string { + t.Helper() + logger := discardLogger() + vs, err := knowledge.NewFileStore(h.vault, reviewrules.VaultName, logger) + if err != nil { + t.Fatal(err) + } + p := knowledge.NewPrimer(nil, knowledge.PrimerConfig{MaxFacts: 25}, logger) + p.AddPinnedFileStore(reviewrules.VaultName, vs, knowledge.LayerProject) + out, _ := p.PrimeForKick(context.Background(), nil, knowledge.KickContext{Lane: lane}, nil).Render() + return out +} + +// The concrete case: the maintainer states that the maintainers file is the +// source of truth. The sweep records it with provenance, tells the reviewer on +// the PR, hands the agent the recorded rule, and the quality agent's next run +// receives it. +func TestReviewFeedbackSweep_MaintainersFileRuleIsCapturedAndRetrievableByQuality(t *testing.T) { + h := newRuleHarness(t, reviewrules.Config{}) + h.feedback(ruleItem("inline:1001", "MEMBER", maintainersFixture)) + res := h.sweep(t).run(context.Background(), parseReviewFeedbackKey) + if res.Kicked != 1 || res.RulesCaptured != 1 { + t.Fatalf("res = %+v, want one kick and one captured rule", res) + } + rules := h.rules.List() + if len(rules) != 1 { + t.Fatalf("rules = %+v", rules) + } + r := rules[0] + p := r.Provenance[0] + if r.ID != "R-1" || r.Text != maintainersRuleText || r.Status != reviewrules.StatusActive || r.AppliesTo != "quality" || + p.CommentURL != fixtureCommentURL || p.Reviewer != "nate-double-u" || p.Association != "MEMBER" || p.PRURL != "https://github.com/cncf/techdocs/pull/424" || p.At.IsZero() { + t.Fatalf("rule = %+v provenance = %+v", r, p) + } + + // The reviewer sees it was captured, where it applies, and can object. + if len(h.src.comments) != 1 { + t.Fatalf("comments = %q, want one acknowledgement", h.src.comments) + } + ack := h.src.comments[0] + for _, want := range []string{"Recorded as rule R-1 in the hive's knowledge, applies to the `quality` agent's future work", maintainersRuleText, fixtureCommentURL, "hivectl knowledge state"} { + if !strings.Contains(ack, want) { + t.Errorf("acknowledgement missing %q:\n%s", want, ack) + } + } + if strings.Contains(ack, "capture this") { + t.Errorf("the capture request is not part of the rule:\n%s", ack) + } + + // The agent is told it is recorded and not to record it again. + brief := h.kick.kicks["quality"][0] + for _, want := range []string{"R-1 (recorded; applies to the `quality` agent's future work)", "do NOT run hivectl knowledge create", "hive:review-rules", fixtureCommentURL} { + if !strings.Contains(brief, want) { + t.Errorf("brief missing %q", want) + } + } + + // Retrievable for the quality agent's next run, with its provenance; not for another agent. + got := h.retrievedFor(t, "quality") + for _, want := range []string{maintainersRuleText, "R-1", fixtureCommentURL, "nate-double-u"} { + if !strings.Contains(got, want) { + t.Errorf("quality knowledge missing %q:\n%s", want, got) + } + } + if other := h.retrievedFor(t, "developer"); strings.Contains(other, maintainersRuleText) { + t.Errorf("a quality-scoped rule reached the developer agent:\n%s", other) + } +} + +// Re-delivery of the same comment (a fresh sweep state, a retried kick) records +// nothing twice and posts no second acknowledgement. +func TestReviewFeedbackSweep_RuleCaptureIsIdempotent(t *testing.T) { + h := newRuleHarness(t, reviewrules.Config{}) + h.feedback(ruleItem("inline:1001", "OWNER", maintainersFixture)) + h.sweep(t).run(context.Background(), parseReviewFeedbackKey) + res := h.sweep(t).run(context.Background(), parseReviewFeedbackKey) // new reviewfeedback state: the comment is routed again + if res.Kicked != 1 || res.RulesCaptured != 0 || len(h.rules.List()) != 1 || len(h.src.comments) != 1 { + t.Fatalf("res = %+v rules = %d comments = %d, want no second capture or acknowledgement", res, len(h.rules.List()), len(h.src.comments)) + } + if !strings.Contains(h.kick.kicks["quality"][1], "R-1 (recorded") { + t.Fatal("a retried kick still names the recorded rule") + } +} + +func TestReviewFeedbackSweep_UntrustedReviewerRuleIsQueuedNotApplied(t *testing.T) { + h := newRuleHarness(t, reviewrules.Config{}) + h.feedback(ruleItem("inline:1001", "CONTRIBUTOR", maintainersFixture)) + res := h.sweep(t).run(context.Background(), parseReviewFeedbackKey) + if res.Kicked != 1 || res.RulesQueued != 1 || res.RulesCaptured != 0 { + t.Fatalf("res = %+v", res) + } + if len(h.src.comments) != 0 { + t.Fatalf("a queued rule is not announced as recorded: %q", h.src.comments) + } + if !strings.Contains(h.kick.kicks["quality"][0], "queued for a person to confirm") { + t.Fatal("the brief must say the rule is queued") + } + if got := h.retrievedFor(t, "quality"); strings.Contains(got, maintainersRuleText) { + t.Fatalf("a queued rule must not reach an agent:\n%s", got) + } +} + +func TestReviewFeedbackSweep_CorrectionsAreNotRules(t *testing.T) { + h := newRuleHarness(t, reviewrules.Config{}) + h.feedback(ruleItem("inline:1", "OWNER", "The Agent readiness score is too high.")) + res := h.sweep(t).run(context.Background(), parseReviewFeedbackKey) + if res.Kicked != 1 || res.RulesCaptured+res.RulesQueued+res.RulesMerged != 0 || len(h.rules.List()) != 0 || len(h.src.comments) != 0 { + t.Fatalf("res = %+v", res) + } + if !strings.Contains(h.kick.kicks["quality"][0], "hivectl knowledge create") { + t.Fatal("with nothing captured the brief keeps the best-effort instruction") + } +} + +func TestReviewFeedbackSweep_NearDuplicateRuleMerges(t *testing.T) { + h := newRuleHarness(t, reviewrules.Config{}) + h.feedback(ruleItem("inline:1001", "OWNER", maintainersFixture)) + h.sweep(t).run(context.Background(), parseReviewFeedbackKey) + h.feedback(ruleItem("inline:1002", "OWNER", "Again: the .project/maintainers.yaml is the source of truth for ownership and maintainership.")) + h.now = h.now.Add(time.Hour) + res := h.sweep(t).run(context.Background(), parseReviewFeedbackKey) + if res.RulesMerged != 1 || len(h.rules.List()) != 1 || len(h.rules.List()[0].Provenance) != 2 { + t.Fatalf("res = %+v rules = %+v", res, h.rules.List()) + } +} + +// A rule is captured even when the burst cannot be kicked: the cap was hit, or +// no agent can take it. +func TestReviewFeedbackSweep_RuleIsCapturedWhenTheKickCannotGo(t *testing.T) { + t.Run("cap reached", func(t *testing.T) { + h := newRuleHarness(t, reviewrules.Config{}) + h.feedback(ruleItem("inline:1001", "OWNER", maintainersFixture)) + s := h.sweep(t) + for i := 0; i < wireTestLimits.WindowCap; i++ { + if err := s.Store.RecordKick(feedbackTestKey, nil, h.now.Add(-time.Duration(i+1)*time.Minute)); err != nil { + t.Fatal(err) + } + } + res := s.run(context.Background(), parseReviewFeedbackKey) + if res.CapReached != 1 || res.RulesCaptured != 1 || len(h.kick.kicks) != 0 { + t.Fatalf("res = %+v", res) + } + }) + t.Run("no agent", func(t *testing.T) { + h := newRuleHarness(t, reviewrules.Config{}) + h.feedback(ruleItem("inline:1001", "OWNER", maintainersFixture)) + s := h.sweep(t) + s.Route = func(string) reviewFeedbackRoute { return reviewFeedbackRoute{Reason: "no agent"} } + res := s.run(context.Background(), parseReviewFeedbackKey) + if res.NeedsHuman != 1 || res.RulesCaptured != 1 || len(h.rules.List()) != 1 { + t.Fatalf("res = %+v", res) + } + }) + t.Run("still debouncing", func(t *testing.T) { + h := newRuleHarness(t, reviewrules.Config{}) + it := ruleItem("inline:1001", "OWNER", maintainersFixture) + it.CreatedAt = h.now + h.feedback(it) + res := h.sweep(t).run(context.Background(), parseReviewFeedbackKey) + if res.Debounced != 1 || len(h.rules.List()) != 0 { + t.Fatalf("a burst still arriving is not captured yet: %+v", res) + } + }) +} + +// A rule whose home is the reviewed project's own files, in a repo the hive does +// not manage: one needs-human note carrying the exact text, once. +func TestReviewFeedbackSweep_ForeignRepoRuleGetsOneProposalNote(t *testing.T) { + h := newRuleHarness(t, reviewrules.Config{}) + h.feedback(ruleItem("inline:1001", "OWNER", "Going forward, add the criteria to your docs: the maintainers file is the source of truth for ownership.")) + h.sweep(t).run(context.Background(), parseReviewFeedbackKey) + h.sweep(t).run(context.Background(), parseReviewFeedbackKey) + h.now = h.now.Add(time.Hour) + h.sweep(t).run(context.Background(), parseReviewFeedbackKey) + var notes int + for _, c := range h.src.comments { + if strings.Contains(c, reviewrules.HomeMarkerPrefix) { + notes++ + for _, want := range []string{"Needs a human", "`cncf/techdocs`", "the maintainers file is the source of truth for ownership", fixtureCommentURL, "does not write to that repo"} { + if !strings.Contains(c, want) { + t.Errorf("proposal note missing %q:\n%s", want, c) + } + } + } + } + if notes != 1 || len(h.src.comments) != 2 { + t.Fatalf("comments = %d (proposal notes %d), want one acknowledgement and one proposal: %q", len(h.src.comments), notes, h.src.comments) + } +} + +func TestReviewFeedbackSweep_ManagedRepoRuleNeedsNoProposal(t *testing.T) { + h := newRuleHarness(t, reviewrules.Config{}) + s := h.sweep(t) + s.Managed = func(string) bool { return true } + h.feedback(ruleItem("inline:1001", "OWNER", "Going forward, add the criteria to your docs: always sign commits.")) + s.run(context.Background(), parseReviewFeedbackKey) + for _, c := range h.src.comments { + if strings.Contains(c, reviewrules.HomeMarkerPrefix) { + t.Fatalf("a managed repo's rule is the hive's own to keep: %s", c) + } + } +} + +type flakyCommentSource struct { + *fakeFeedbackSource + fail bool +} + +func (f *flakyCommentSource) CreateIssueComment(ctx context.Context, repo string, n int, body string) error { + if f.fail { + return errors.New("secondary rate limit") + } + return f.fakeFeedbackSource.CreateIssueComment(ctx, repo, n, body) +} + +// A notice that could not be posted is retried on a later pass, even when the +// PR has no new feedback, and is posted once. +func TestReviewFeedbackSweep_FailedNoticesAreRetried(t *testing.T) { + h := newRuleHarness(t, reviewrules.Config{}) + flaky := &flakyCommentSource{fakeFeedbackSource: h.src, fail: true} + h.feedback(ruleItem("inline:1001", "OWNER", "Going forward, add the criteria to your docs: the maintainers file is the source of truth.")) + s := h.sweep(t) + s.Source = flaky + s.run(context.Background(), parseReviewFeedbackKey) + if len(h.src.comments) != 0 { + t.Fatalf("comments = %q while posting fails", h.src.comments) + } + flaky.fail = false + h.feedback() // nothing new from people + s.run(context.Background(), parseReviewFeedbackKey) + s.run(context.Background(), parseReviewFeedbackKey) + if len(h.src.comments) != 2 { + t.Fatalf("comments = %q, want the acknowledgement and the proposal once each", h.src.comments) + } +} + +// The agent's structured block sharpens a rule: it is read from the agent's own +// reply even when no human comment is waiting, applied once, and must cite a +// comment the hive saw. +func TestReviewFeedbackSweep_AgentRuleBlockRefinesTheRule(t *testing.T) { + h := newRuleHarness(t, reviewrules.Config{}) + h.feedback(ruleItem("inline:1001", "OWNER", maintainersFixture)) + h.sweep(t).run(context.Background(), parseReviewFeedbackKey) + + block := "Updated.\n" + fb := openDraftFeedback() + fb.Items = nil + fb.HiveReplies = []github.FeedbackMessage{{ID: "conversation:77", Author: "hive", Hive: true, Body: block}} + h.src.byKey[feedbackTestKey] = fb + h.now = h.now.Add(time.Hour) + for i := 0; i < 2; i++ { + h.sweep(t).run(context.Background(), parseReviewFeedbackKey) + } + r := h.rules.Get("R-1") + if r.Scope != reviewrules.ScopeWorkflow || r.AppliesTo != "cncf-techdocs-assessment" || !strings.Contains(r.Text, "For every project assessment") { + t.Fatalf("rule = %+v", r) + } + if len(h.rules.List()) != 1 { + t.Fatal("the entry citing a comment the hive never saw must not create a rule") + } + // The workflow-scoped rule reaches that workflow's lane and no longer the agent lane. + if got := h.retrievedFor(t, "cncf-techdocs-assessment"); !strings.Contains(got, "For every project assessment") { + t.Fatalf("workflow lane knowledge:\n%s", got) + } +} + +func TestReviewFeedbackSweep_RulesOffChangesNothing(t *testing.T) { + h := newRuleHarness(t, reviewrules.Config{}) + h.feedback(ruleItem("inline:1001", "OWNER", maintainersFixture)) + s := h.sweep(t) + s.Rules = nil + res := s.run(context.Background(), parseReviewFeedbackKey) + if res.Kicked != 1 || len(h.src.comments) != 0 || !strings.Contains(h.kick.kicks["quality"][0], "hivectl knowledge create") { + t.Fatalf("res = %+v comments = %q", res, h.src.comments) + } +} + +func TestReviewFeedbackSweep_RuleLimitRefusalIsLoggedNotFatal(t *testing.T) { + h := newRuleHarness(t, reviewrules.Config{MaxRules: 1}) + h.feedback(ruleItem("inline:1001", "OWNER", maintainersFixture), ruleItem("inline:1002", "OWNER", "Going forward, cite the scorecard with a commit-pinned link.")) + res := h.sweep(t).run(context.Background(), parseReviewFeedbackKey) + if res.Kicked != 1 || res.RulesCaptured != 1 || len(h.rules.List()) != 1 { + t.Fatalf("res = %+v", res) + } +} + +func TestRecordedRulesBriefShape(t *testing.T) { + brief := reviewfeedback.BuildBriefWithRules("o/r", 1, "b", "", []github.FeedbackItem{ruleItem("inline:1", "OWNER", "x")}, + []reviewfeedback.RecordedRule{{ID: "R-2", Text: "t", Applies: "all agents' work", Queued: true, SourceURL: "u"}}) + if !strings.Contains(brief, "R-2 (queued for a person to confirm; applies to all agents' work): t [source u]") || strings.Contains(brief, "\u2014") { + t.Fatalf("brief:\n%s", brief) + } +} diff --git a/src/cmd/hive/review_feedback_wire.go b/src/cmd/hive/review_feedback_wire.go index 22f5c16fd2..34ef56b2df 100644 --- a/src/cmd/hive/review_feedback_wire.go +++ b/src/cmd/hive/review_feedback_wire.go @@ -12,6 +12,7 @@ import ( "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/reviewfeedback" + "github.com/hivecommons/hive/pkg/reviewrules" ) // Review-feedback response sweep (hivecommons/hive#19027). A person's review @@ -36,6 +37,7 @@ var ( reviewFeedbackLastRun time.Time reviewFeedbackCursor int reviewFeedbackStore *reviewfeedback.Store + reviewRuleStore *reviewrules.Store ) // reviewFeedbackSource is the GitHub surface the sweep needs. @@ -66,11 +68,20 @@ type reviewFeedbackSweep struct { Cursor int Now time.Time Logger *slog.Logger + // Rules captures reviewer-stated rules into the hive's knowledge + // (hivecommons/hive#19046). Nil turns capture off; the sweep is + // otherwise unchanged. + Rules *reviewrules.Store + // Lane maps the authoring agent to the base lane its rules apply to. + Lane func(owner string) string + // Managed says whether the hive manages a repo (owner/name). + Managed func(repo string) bool } // reviewFeedbackResult counts one pass, for tests and the summary line. type reviewFeedbackResult struct { Checked, Kicked, Debounced, CapReached, NeedsHuman, Errors int + RulesCaptured, RulesQueued, RulesMerged int NextCursor int } @@ -108,6 +119,10 @@ func (s reviewFeedbackSweep) run(ctx context.Context, parse func(key string) (re s.Logger.Warn("review feedback: fetch failed, PR skipped this pass", "repo", repo, "pr", number, "error", err) continue } + if s.Rules != nil && fb.Open() { + s.applyRuleReplies(repo, number, fb.HiveReplies) + s.postRuleNotices(ctx, repo, number) + } if !fb.Open() || len(fb.Items) == 0 { continue } @@ -118,13 +133,15 @@ func (s reviewFeedbackSweep) run(ctx context.Context, parse func(key string) (re s.Logger.Debug("review feedback: burst still arriving, waiting to coalesce", "repo", repo, "pr", number, "comments", len(d.Items)) case reviewfeedback.VerdictCapReached: res.CapReached++ + s.captureRules(ctx, repo, number, redPRFixOwner(s.Owners[key]), d.Items, &res) s.needsHuman(ctx, key, repo, number, "it has been routed to its agent "+strconv.Itoa(s.Limits.WindowCap)+" times in "+s.Limits.Window.String()+"; further feedback is waiting, and a new comment from a person resets this", d.Items, &res) case reviewfeedback.VerdictKick: + recorded := s.captureRules(ctx, repo, number, redPRFixOwner(s.Owners[key]), d.Items, &res) if route.Agent == "" { s.needsHuman(ctx, key, repo, number, route.Reason, d.Items, &res) continue } - brief := reviewfeedback.BuildBrief(repo, number, fb.HeadRef, s.WritingGuide, d.Items) + brief := reviewfeedback.BuildBriefWithRules(repo, number, fb.HeadRef, s.WritingGuide, d.Items, recorded) if err := s.Kicker.Kick(route.Agent, brief); err != nil { level := slog.LevelWarn if isSharedPoolBusyErr(err) { @@ -161,6 +178,117 @@ func (s reviewFeedbackSweep) needsHuman(ctx context.Context, key, repo string, n res.NeedsHuman++ } +// captureRules is the deterministic half of rule capture (#19046): every +// comment in the burst that states a rule is written to the hive's knowledge +// store whether or not the agent later records it, idempotently per comment. +// It runs for every verdict that carries comments, so a rule is not lost to the +// cap or to an unavailable agent. It returns the rules for the agent's brief. +func (s reviewFeedbackSweep) captureRules(ctx context.Context, repo string, number int, owner string, items []github.FeedbackItem, res *reviewFeedbackResult) []reviewfeedback.RecordedRule { + if s.Rules == nil { + return nil + } + lane := owner + if s.Lane != nil { + lane = s.Lane(owner) + } + managed := s.Managed != nil && s.Managed(repo) + var recorded []reviewfeedback.RecordedRule + for _, it := range items { + c := reviewrules.Comment{Repo: repo, PR: number, CommentID: it.ID, URL: it.URL, Reviewer: it.Author, + Association: it.AuthorAssociation, Body: it.Body, At: it.CreatedAt, Lane: lane, ManagedRepo: managed} + if err := s.Rules.Remember(c, s.Now); err != nil { + s.Logger.Warn("review feedback: could not remember the comment for rule evidence", "repo", repo, "pr", number, "comment", it.ID, "error", err) + } + r := s.Rules.CaptureComment(c, s.Now) + switch r.Outcome { + case reviewrules.OutcomeNotRule: + continue + case reviewrules.OutcomeRefused: + s.Logger.Warn("review feedback: rule not recorded", "repo", repo, "pr", number, "comment", it.ID, "reviewer", it.Author, "signals", r.Signals, "error", r.Err) + continue + case reviewrules.OutcomeCaptured: + res.RulesCaptured++ + case reviewrules.OutcomeQueued: + res.RulesQueued++ + case reviewrules.OutcomeMerged: + res.RulesMerged++ + } + if r.Outcome != reviewrules.OutcomeDuplicate { + s.Logger.Info("review feedback: reviewer-stated rule recorded in the hive's knowledge", "repo", repo, "pr", number, "comment", it.ID, + "reviewer", it.Author, "association", it.AuthorAssociation, "outcome", string(r.Outcome), "rule", r.Rule.ID, "scope", string(r.Rule.Scope), + "applies_to", r.Rule.AppliesTo, "signals", r.Signals) + } + if r.Rule == nil || (r.Rule.Status != reviewrules.StatusActive && r.Rule.Status != reviewrules.StatusPending) { + continue + } + recorded = append(recorded, reviewfeedback.RecordedRule{ID: r.Rule.ID, Text: r.Rule.Text, Applies: r.Rule.Describe(), + Queued: r.Rule.Status == reviewrules.StatusPending, SourceURL: it.URL}) + } + s.postRuleNotices(ctx, repo, number) + return recorded +} + +// applyRuleReplies reads the structured block in the agent's own replies on the +// PR. Each reply is applied once; an entry that does not validate, or does not +// cite a comment the hive saw, is logged and dropped. +func (s reviewFeedbackSweep) applyRuleReplies(repo string, number int, replies []github.FeedbackMessage) { + for _, rp := range replies { + applied := s.Rules.ApplyReply(reviewfeedback.Key(repo, number)+":"+rp.ID, rp.Body, s.Managed, s.Now) + for _, a := range applied { + if a.Outcome == reviewrules.OutcomeRejected { + s.Logger.Warn("review feedback: agent rule entry rejected", "repo", repo, "pr", number, "reply", rp.ID, "reason", a.Reason) + continue + } + s.Logger.Info("review feedback: agent rule block applied", "repo", repo, "pr", number, "reply", rp.ID, "outcome", string(a.Outcome), "rule", a.Rule.ID) + } + } +} + +// postRuleNotices tells reviewers what was captured and posts the one +// needs-human proposal for a rule whose home is another project's repo. +func (s reviewFeedbackSweep) postRuleNotices(ctx context.Context, repo string, number int) { + if acks := s.Rules.UnackedFor(repo, number); len(acks) > 0 { + if s.acknowledgeRules(ctx, repo, number, reviewrules.AckComment(acks)) { + ids := make([]string, 0, len(acks)) + for _, a := range acks { + ids = append(ids, a.CommentID) + } + if err := s.Rules.MarkAcked(repo, number, ids); err != nil { + s.Logger.Warn("review feedback: could not persist rule acknowledgement", "repo", repo, "pr", number, "error", err) + } + } + } + for _, r := range s.Rules.HomeProposals() { + if len(r.Provenance) == 0 || r.Provenance[0].Repo != repo || r.Provenance[0].PR != number { + continue + } + if s.proposeRuleHome(ctx, repo, number, reviewrules.HomeProposalNote(r)) { + if err := s.Rules.MarkHomeNoted(r.ID); err != nil { + s.Logger.Warn("review feedback: could not persist rule proposal flag", "rule", r.ID, "error", err) + } + } + } +} + +// acknowledgeRules posts the reply that says which rules were recorded. +func (s reviewFeedbackSweep) acknowledgeRules(ctx context.Context, repo string, number int, body string) bool { + if err := s.Source.CreateIssueComment(ctx, repo, number, body); err != nil { // writeback:solicited a person's review feedback on this hive-authored PR + s.Logger.Warn("review feedback: rule acknowledgement not posted, retrying next pass", "repo", repo, "pr", number, "error", err) + return false + } + return true +} + +// proposeRuleHome posts the one needs-human note carrying the proposed text of +// a rule that belongs in a repo the hive does not manage. +func (s reviewFeedbackSweep) proposeRuleHome(ctx context.Context, repo string, number int, body string) bool { + if err := s.Source.CreateIssueComment(ctx, repo, number, body); err != nil { // writeback:solicited a person's review feedback on this hive-authored PR + s.Logger.Warn("review feedback: rule proposal note not posted, retrying next pass", "repo", repo, "pr", number, "error", err) + return false + } + return true +} + // parseReviewFeedbackKey splits "owner/repo#n". func parseReviewFeedbackKey(key string) (string, int, bool) { i := strings.LastIndex(key, "#") @@ -197,6 +325,18 @@ func sweepReviewFeedback(ctx context.Context, cfg *config.Config, client *github } reviewFeedbackStore = st } + if reviewRuleStore == nil && reviewrules.Enabled() { + rs, err := reviewrules.Open(reviewrules.Dir(), reviewrules.VaultDir(), reviewrules.ConfigFromEnv()) + if err != nil { + logger.Warn("review feedback: rule index unreadable or not writable; rules kept in memory only", "dir", reviewrules.Dir(), "error", err) + } + reviewRuleStore = rs + } + if reviewRuleStore != nil { + for _, line := range reviewRuleStore.Reconcile(now) { + logger.Info("review feedback: rule changed by an operator", "change", line) + } + } available := agentAvailability(cfg, agentMgr) sweep := reviewFeedbackSweep{ Source: client, @@ -209,6 +349,9 @@ func sweepReviewFeedback(ctx context.Context, cfg *config.Config, client *github Cursor: reviewFeedbackCursor, Now: now, Logger: logger, + Rules: reviewRuleStore, + Lane: cfg.BaseAgentName, + Managed: cfg.IsManagedRepo, Route: func(owner string) reviewFeedbackRoute { if ac, ok := cfg.Agents[owner]; (agentMgr != nil && agentMgr.IsPaused(owner)) || (ok && ac.Paused) { return reviewFeedbackRoute{Paused: true} @@ -229,6 +372,7 @@ func sweepReviewFeedback(ctx context.Context, cfg *config.Config, client *github reviewFeedbackCursor = res.NextCursor if res.Kicked+res.NeedsHuman+res.Debounced+res.CapReached+res.Errors > 0 { logger.Info("review feedback pass", "checked", res.Checked, "kicked", res.Kicked, "debounced", res.Debounced, - "cap_reached", res.CapReached, "needs_human", res.NeedsHuman, "errors", res.Errors) + "cap_reached", res.CapReached, "needs_human", res.NeedsHuman, "errors", res.Errors, + "rules_captured", res.RulesCaptured, "rules_queued", res.RulesQueued, "rules_merged", res.RulesMerged) } } diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index 4190e8a512..661b70b1d3 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -106,6 +106,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [Native merge queue](native-merge-queue.md) — how the auto-merge lanes enqueue eligible PRs into GitHub's native merge queue (App auth only), the `native-merge-queued`/`native-merge-queue-*` stage reasons, and how Hive withdraws only its own entries when a gate breaks. - [`hive-open-issue`](hive-open-issue.md) — **Agent/developer reference:** request issue creation, comments, or claims through the App bot, with deduplication and retries. - [Review-bot threads on hive-mediated PRs](review-bot-threads.md) — route bot feedback back to the authoring agent while protecting human review threads. +- [Reviewer-stated rules from review feedback](review-feedback-rules.md): how the review-feedback sweep captures rules and sources of truth a person states on a hive PR into the knowledge store, gates them on reviewer trust, loads them into the matching agent's next runs, and proposes the text when its home is another project's repo. - [Network and port requirements](network-requirements.md) — inbound ports, proxy paths, egress, and firewall guidance. - [TLS, HTTPS, and certificates](tls-setup.md) — termination patterns and certificate ownership. - [Security notes](security.md) — log scrubbing and secret redaction guarantees/limits. diff --git a/src/docs/knowledge-entries.md b/src/docs/knowledge-entries.md index 6ee6a5bdfe..ec21b76264 100644 --- a/src/docs/knowledge-entries.md +++ b/src/docs/knowledge-entries.md @@ -113,3 +113,9 @@ touch. The rules themselves (do not run the suite locally, never run it in an agent pane, fragment required, `--fix` first) stay in `AGENTS.md`; only the facts and reasons moved. + +## Reviewer-stated rules + +Rules that people state in review feedback on the hive's own PRs are written as entries of a local `review-rules` channel, tagged `pin`, +scoped to the authoring agent's lane, with the source comment as `source`. A pinned, approved entry whose scope admits a kick is primed +whatever the kick's keywords are. See [review-feedback-rules.md](review-feedback-rules.md). diff --git a/src/docs/labels-and-control-signals/reference.md b/src/docs/labels-and-control-signals/reference.md index e76636ea44..1a19c9d585 100644 --- a/src/docs/labels-and-control-signals/reference.md +++ b/src/docs/labels-and-control-signals/reference.md @@ -119,7 +119,7 @@ The labels the hive applies instead of `hold` ([#18468](https://github.com/hivec | `ready-for-review` | Hive's work is done; a maintainer reviews, then removes this label to let it merge. | The ACMM level gate (L3-L5, and cross-repo outreach at every level). | | `decision-needed` | Hive is blocked on a human decision; a maintainer decides (see hive's comment). | The #5117 self-authorization gate, the #9665 reporter-trust gate (with `needs-human`), the holdguard re-hold, and an owner-only delegated decision (with `needs-owner`). | -Every gate posts its existing notice comment saying why. Agents cannot add or remove either label through the relay. Neither label stops the hive from answering review feedback: when a person leaves an inline comment, a review (`COMMENTED` or `CHANGES_REQUESTED`) or a comment addressed to the hive on a PR the hive's own agent opened, that agent is kicked to verify the claim and push a signed revision or reply with evidence, whether the PR is a draft, held, or in a repo the hive does not manage ([#19027](https://github.com/hivecommons/hive/issues/19027); `src/cmd/hive/review_feedback_wire.go`, `src/pkg/reviewfeedback/reviewfeedback.go`). A hold means do not merge, not do not respond. A burst of comments is coalesced into one kick after `HIVE_REVIEW_FEEDBACK_DEBOUNCE` (default 3m); comments that arrive while the agent works get their own follow-up kick; each PR is capped at `HIVE_REVIEW_FEEDBACK_WINDOW_CAP` kicks (default 12) per `HIVE_REVIEW_FEEDBACK_WINDOW` (default 24h), the cap posts a needs-human note, and a person's next comment resets it; `HIVE_REVIEW_FEEDBACK_MAX_ITEMS` (default 30) bounds one brief. Set `HIVE_REVIEW_FEEDBACK_RESPONSE=off` to disable the sweep. +Every gate posts its existing notice comment saying why. Agents cannot add or remove either label through the relay. Neither label stops the hive from answering review feedback: when a person leaves an inline comment, a review (`COMMENTED` or `CHANGES_REQUESTED`) or a comment addressed to the hive on a PR the hive's own agent opened, that agent is kicked to verify the claim and push a signed revision or reply with evidence, whether the PR is a draft, held, or in a repo the hive does not manage ([#19027](https://github.com/hivecommons/hive/issues/19027); `src/cmd/hive/review_feedback_wire.go`, `src/pkg/reviewfeedback/reviewfeedback.go`). A hold means do not merge, not do not respond. A burst of comments is coalesced into one kick after `HIVE_REVIEW_FEEDBACK_DEBOUNCE` (default 3m); comments that arrive while the agent works get their own follow-up kick; each PR is capped at `HIVE_REVIEW_FEEDBACK_WINDOW_CAP` kicks (default 12) per `HIVE_REVIEW_FEEDBACK_WINDOW` (default 24h), the cap posts a needs-human note, and a person's next comment resets it; `HIVE_REVIEW_FEEDBACK_MAX_ITEMS` (default 30) bounds one brief. Set `HIVE_REVIEW_FEEDBACK_RESPONSE=off` to disable the sweep. A comment that states a rule or a source of truth for future work is also captured into the hive's knowledge with its provenance, gated on the reviewer's association, and acknowledged on the PR ([#19046](https://github.com/hivecommons/hive/issues/19046); see [review-feedback-rules.md](../review-feedback-rules.md)). **Migration.** On boot, Hive creates both labels (with their descriptions) in each active repository and swaps a literal `hold` for the matching gate label only when the newest `hold` event is a `labeled` by the Hive App **and** an App-authored gate notice is on the thread (level-hold notice → `ready-for-review`; reporter-trust, #5117, holdguard or owner-only decision notice → `decision-needed`). It adds the gate label before removing `hold` and leaves one `` comment. A `hold` a person applied (or re-applied) is never touched. Bounded to 50 items per repository per boot (`src/pkg/github/hive_gate_label_migration.go`). diff --git a/src/docs/notifications.md b/src/docs/notifications.md index 3ec7164edb..cdccd2bf2e 100644 --- a/src/docs/notifications.md +++ b/src/docs/notifications.md @@ -508,6 +508,7 @@ without a class. Review submissions and review requests are always counted as | `github.handleMergedPRIssueStage` | `status` | | | `github.runRecheckSoak` | `status` | | | `main.postReviewAcknowledgements` | `status` | | +| `main.reviewFeedbackSweep.acknowledgeRules` | `status` | | | `main.rfcNotifySource.Comment` | `status` | | | `main.unparkEscalatedPR` | `status` | | | `dashboard.(*ContributeWSHub).parkWrongBase` | `attention` | | @@ -537,6 +538,7 @@ without a class. Review submissions and review requests are always counted as | `main.rfcCoverageForge.CreateComment` | `attention` | | | `main.runEscalationSweep` | `attention` | | | `main.reviewFeedbackSweep.needsHuman` | `attention` | | +| `main.reviewFeedbackSweep.proposeRuleHome` | `attention` | | | `prfollowup.postStaleHeadPushNotes` | `attention` | | | `scheduler.(*Scheduler).judgePriorArt` | `attention` | | | `scheduler.(*Scheduler).parkUnclearIssue` | `attention` | | diff --git a/src/docs/review-feedback-rules.md b/src/docs/review-feedback-rules.md new file mode 100644 index 0000000000..a77ed810f2 --- /dev/null +++ b/src/docs/review-feedback-rules.md @@ -0,0 +1,82 @@ +# Reviewer-stated rules from review feedback + +When a person reviews a PR one of the hive's agents opened, most comments are corrections: they apply to that PR and the agent fixes it +([#19027](https://github.com/hivecommons/hive/issues/19027)). Some comments state a rule or a source of truth meant to hold for later work, +for example "`.project/maintainers.yaml` is the source of truth for ownership and maintainership; capture this in the assessment knowledge +base". A fix does not help the next analysis. The review-feedback sweep therefore captures such rules itself, deterministically, instead of +leaving it to the agent's discretion ([#19046](https://github.com/hivecommons/hive/issues/19046)). + +## What happens + +1. **Classify.** Each comment in a coalesced burst is matched against fixed phrase signals. A comment with at least one is a rule; the rest + are corrections and are handled as before. The signals are `source-of-truth`, `authoritative`, `always` ("should always", "must always"), + `never` ("should never", "must never"), `going-forward` ("going forward", "from now on", "next time", "for future assessments"), + `capture-request` ("capture this", "document this", "record it"), `assessment-criteria` ("assessment criteria", "assessment knowledge") + and `knowledge-base`. Comment text is data: it is matched and quoted, never executed, fetched or followed. +2. **Extract.** The rule text is the sentences that carry a signal, without clauses that only ask for the capture, bounded to 600 characters. +3. **Gate on trust.** A rule changes what every later run is told, so only reviewers whose GitHub association is in + `HIVE_REVIEW_FEEDBACK_RULE_TRUSTED_ASSOCIATIONS` (default `OWNER,MEMBER,COLLABORATOR`) are captured at once. Any other reviewer's rule is + queued as a draft knowledge entry that a person approves. A trusted reviewer restating a queued rule activates it. +4. **Persist.** The rule gets an id (`R-1`, `R-2`, ...) and is written to the `review-rules` local knowledge channel as an entry tagged `pin`, + with provenance: the PR, the comment id and URL (the entry's `source`), the reviewer, their association, the time, and whether the hive or + the agent wrote the text. The same comment id is never captured twice, across restarts too. A restatement with near-identical text and the + same scope (token overlap at or above `HIVE_REVIEW_FEEDBACK_RULE_MERGE_THRESHOLD`, default 0.8) merges into the existing rule and adds its + comment to the provenance. At most `HIVE_REVIEW_FEEDBACK_RULES_MAX` (default 200) rules are live; past that, capture is refused and logged. +5. **Say so.** The sweep posts one reply on the PR: "Recorded as rule R-n in the hive's knowledge, applies to ...", with the recorded text, the + source comment and how to retract it, so the reviewer sees it was captured and can object. The agent's brief lists the recorded rules, tells + it not to record them again, and asks it to name the ids in its reply. +6. **Load.** The kick primer carries every approved `pin` entry whose scope admits the kick, whatever the kick's keywords are, ahead of ranked + knowledge and exempt from the max-facts trim (at most 20 pinned entries per kick). A rule applies to the authoring agent's lane by default, + so the quality agent receives its rules on its next kick; a rule scoped to a workflow reaches the lane or workflow model of that name. The + usual lifecycle filter and `knowledge.agent_scopes` still apply, so drafts, retracted rules and out-of-scope agents never see a rule. + +Capture happens when the burst is routed, so it also happens when the per-PR cap was reached or no agent could take the kick. It does not +change the sweep's debounce, cap or thread behavior. + +## The agent's structured block + +The kicked agent may end its reply with one hidden line to sharpen a rule, or to state one the classifier missed: + +```text + +``` + +`scope` is `agent` (default, the authoring agent), `workflow` (`applies_to` names it), `repo` (`applies_to` is `owner/name`) or `global`. +Every entry is validated field by field: no unknown fields, bounded text, a GitHub PR or issue URL as `evidence_url`, a restricted +`applies_to`, at most 10 rules per reply. `evidence_url` must name a review comment the hive saw: the reviewer's trust comes from that +comment, never from the block, so an agent cannot raise a rule's authority. A reply is applied once. If a refined rule becomes a near +duplicate of another, the two merge and the older id survives. A `repo` rule is scoped to its lane and tagged `repo:`; it is +not pinned, so it reaches kicks by relevance. + +## Rules that belong in someone else's repo + +The hive does not write to repos it does not manage. When a rule's proper home is the reviewed project's own files (the reviewer asks for it +"in your docs" or "in your repo" on a repo the hive does not manage, or the agent's block gives a `repo` scope for such a repo), the hive still +records it in its own knowledge and posts exactly one needs-human note per rule on the PR. The note carries the exact proposed text and the +source comment, so a maintainer can place it. It is posted once and retried on a later pass if posting failed. + +## Seeing and retracting rules + +Rules are ordinary knowledge entries in the `review-rules` channel (`/data/vaults/review-rules`, override with +`HIVE_REVIEW_FEEDBACK_RULES_VAULT`), so the dashboard's knowledge list shows each rule with its source and the provenance lines in its body. +The rule index lives in the sweep's state directory (`HIVE_REVIEW_FEEDBACK_DIR`, file `rules.json`). The channel is connected at boot when the +vaults directory exists; the primer reads it within the vault's usual refresh interval after a change. + +- Retract a rule: `hivectl knowledge state rule-r-1 deprecated`. Deleting the entry file does the same. The sweep reconciles on its next pass + (every five minutes) and records the change in its log. +- Confirm a queued rule: `hivectl knowledge state rule-r-2 approved`. +- Capture log lines: `review feedback: reviewer-stated rule recorded in the hive's knowledge`, with the rule, reviewer, association, scope and + the signals that matched. + +## Settings + +| Variable | Default | Meaning | +|---|---|---| +| `HIVE_REVIEW_FEEDBACK_RULES` | on | Set to `0`, `false`, `off` or `no` to stop rule capture. The sweep keeps running. | +| `HIVE_REVIEW_FEEDBACK_RULE_TRUSTED_ASSOCIATIONS` | `OWNER,MEMBER,COLLABORATOR` | Associations whose rules are captured at once; others are queued. | +| `HIVE_REVIEW_FEEDBACK_RULES_MAX` | `200` | Bound on live (active or queued) rules. | +| `HIVE_REVIEW_FEEDBACK_RULE_MERGE_THRESHOLD` | `0.8` | Text similarity at which same-scope rules merge. | +| `HIVE_REVIEW_FEEDBACK_RULES_VAULT` | `/data/vaults/review-rules` | Knowledge channel the rules are written to. | +| `HIVE_REVIEW_FEEDBACK_DIR` | `/data/review-feedback` | State directory shared with the sweep. | + +Implementation: `src/pkg/reviewrules/`, `src/cmd/hive/review_feedback_wire.go`, and the pinned-entry support in `src/pkg/knowledge/primer_pinned.go`. diff --git a/src/pkg/github/notification_write_sites.go b/src/pkg/github/notification_write_sites.go index e09daf832a..28b6f15dc8 100644 --- a/src/pkg/github/notification_write_sites.go +++ b/src/pkg/github/notification_write_sites.go @@ -35,6 +35,7 @@ var notificationWriteSites = map[string]NotificationWriteSite{ "automerge.(*engineStatusForge).CreateComment": {Class: NotificationClassStatus, Passthrough: true}, // status: bookkeeping readable on demand. + "main.reviewFeedbackSweep.acknowledgeRules": {Class: NotificationClassStatus}, "dashboard.(*ContributeWSHub).markAlreadyDoneIssue": {Class: NotificationClassStatus}, "dashboard.(*ContributeWSHub).releaseGateParkLabel": {Class: NotificationClassStatus}, "dashboard.(*ContributeWSHub).releaseToolingBlockedLabel": {Class: NotificationClassStatus}, @@ -100,6 +101,7 @@ var notificationWriteSites = map[string]NotificationWriteSite{ "main.rfcCoverageForge.CreateComment": {Class: NotificationClassAttention}, "main.runEscalationSweep": {Class: NotificationClassAttention}, "main.reviewFeedbackSweep.needsHuman": {Class: NotificationClassAttention}, + "main.reviewFeedbackSweep.proposeRuleHome": {Class: NotificationClassAttention}, "prfollowup.postStaleHeadPushNotes": {Class: NotificationClassAttention}, "scheduler.(*Scheduler).judgePriorArt": {Class: NotificationClassAttention}, "scheduler.(*Scheduler).parkUnclearIssue": {Class: NotificationClassAttention}, diff --git a/src/pkg/github/pr_feedback.go b/src/pkg/github/pr_feedback.go index d62a19d034..3555d4de56 100644 --- a/src/pkg/github/pr_feedback.go +++ b/src/pkg/github/pr_feedback.go @@ -70,6 +70,10 @@ type FeedbackItem struct { Line int `json:"line,omitempty"` URL string `json:"url,omitempty"` CreatedAt time.Time `json:"created_at"` + // AuthorAssociation is GitHub's association of the author with the repo + // (OWNER, MEMBER, COLLABORATOR, CONTRIBUTOR), upper-case. The rule + // capture trust gate (pkg/reviewrules) reads it. + AuthorAssociation string `json:"author_association,omitempty"` } // FeedbackMessage is one comment in a review thread. @@ -92,6 +96,10 @@ type PRFeedback struct { Items []FeedbackItem // Source is "graphql" or "rest": which read produced this result. Source string + // HiveReplies are the hive's own comments that carry a review-rules + // block (the structured statement of the rules a reviewer stated). They + // are read even when no human comment is unanswered. + HiveReplies []FeedbackMessage } // Open reports whether the PR can still take a revision. @@ -166,6 +174,9 @@ func (c *Client) FetchPRFeedback(ctx context.Context, repo string, number int) ( HeadRef: raw.HeadRefName, Items: UnansweredFeedback(raw.rawPRConversation, c.isHumanFeedbackAuthor, c.isAgentCommentAuthor, c.addressesHive), Source: source, + HiveReplies: HiveRuleReplies(raw.rawPRConversation, func(a *rawPRCommentAuthor) bool { + return a != nil && c.isHiveLogin(strings.TrimSpace(a.Login)) + }), }, nil } @@ -251,7 +262,8 @@ func UnansweredFeedback(raw rawPRConversation, isHuman func(*rawPRCommentAuthor) return } out = append(out, FeedbackItem{ID: id, ThreadID: thread, Thread: ctxThread, Kind: kind, Author: rc.Author.Login, - Body: truncateCommentBody(rc.Body), Path: path, Line: line, URL: rc.URL, CreatedAt: rc.CreatedAt}) + Body: truncateCommentBody(rc.Body), Path: path, Line: line, URL: rc.URL, CreatedAt: rc.CreatedAt, + AuthorAssociation: strings.ToUpper(rc.AuthorAssociation)}) } for _, rc := range raw.Comments.Nodes { if human(rc) && addressed != nil && addressed(rc.Body) { diff --git a/src/pkg/github/pr_feedback_replies.go b/src/pkg/github/pr_feedback_replies.go new file mode 100644 index 0000000000..fa7a06c160 --- /dev/null +++ b/src/pkg/github/pr_feedback_replies.go @@ -0,0 +1,40 @@ +package github + +import ( + "sort" + "strconv" + "strings" +) + +// FeedbackRulesMarker is the substring that marks a hive reply as carrying a +// structured review-rules block (pkg/reviewrules.BlockMarker holds the same +// value; pkg/github does not import it). +const FeedbackRulesMarker = "hive:review-rules" + +// HiveRuleReplies returns the hive's own comments on a PR, in every place a +// comment lives, whose body carries a review-rules block. isHive must accept +// only the hive's own logins: the block is read as the agent's statement, so a +// comment merely quoting the marker from a person is not one. +func HiveRuleReplies(raw rawPRConversation, isHive func(*rawPRCommentAuthor) bool) []FeedbackMessage { + var out []FeedbackMessage + add := func(kind PRCommentKind, rc rawPRComment) { + if rc.Author == nil || !isHive(rc.Author) || !strings.Contains(rc.Body, FeedbackRulesMarker) { + return + } + out = append(out, FeedbackMessage{ID: string(kind) + ":" + strconv.FormatInt(rc.DatabaseID, 10), Author: rc.Author.Login, + Hive: true, Body: rc.Body, URL: rc.URL, CreatedAt: rc.CreatedAt}) + } + for _, rc := range raw.Comments.Nodes { + add(PRCommentConversation, rc) + } + for _, rc := range raw.Reviews.Nodes { + add(PRCommentReview, rc) + } + for _, th := range raw.ReviewThreads.Nodes { + for _, rc := range th.Comments.Nodes { + add(PRCommentInline, rc) + } + } + sort.SliceStable(out, func(i, j int) bool { return out[i].CreatedAt.Before(out[j].CreatedAt) }) + return out +} diff --git a/src/pkg/github/pr_feedback_replies_test.go b/src/pkg/github/pr_feedback_replies_test.go new file mode 100644 index 0000000000..28eebe620f --- /dev/null +++ b/src/pkg/github/pr_feedback_replies_test.go @@ -0,0 +1,51 @@ +package github + +import ( + "strings" + "testing" + "time" +) + +func TestUnansweredFeedbackCarriesTheAuthorAssociation(t *testing.T) { + c := humanCommentTestClient(t, "http://127.0.0.1:1") + at := time.Date(2026, 11, 18, 15, 0, 0, 0, time.UTC) + var raw rawPRConversation + raw.ReviewThreads.Nodes = []rawPRCommentThread{{ID: "T", Comments: struct { + Nodes []rawPRComment `json:"nodes"` + }{Nodes: []rawPRComment{ + rawComment(1, "nate", "User", "member", "a", at), + rawComment(2, "kim", "User", "CONTRIBUTOR", "b", at.Add(time.Minute)), + }}}} + got := detect(c, raw) + if len(got) != 2 || got[0].AuthorAssociation != "MEMBER" || got[1].AuthorAssociation != "CONTRIBUTOR" { + t.Fatalf("items = %+v, want upper-case associations carried through", got) + } +} + +func TestHiveRuleRepliesReadsOnlyTheHivesOwnBlocks(t *testing.T) { + at := time.Date(2026, 11, 18, 16, 0, 0, 0, time.UTC) + block := "done\n" + isHive := func(a *rawPRCommentAuthor) bool { return a.Login == "hive[bot]" } + var raw rawPRConversation + raw.Comments.Nodes = []rawPRComment{ + rawComment(3, "hive[bot]", "Bot", "NONE", block, at.Add(2*time.Minute)), + rawComment(4, "hive[bot]", "Bot", "NONE", "no block here", at), + rawComment(5, "nate", "User", "OWNER", block, at), // a person quoting the marker is not the agent + {DatabaseID: 6, Body: block, CreatedAt: at}, // deleted account + } + raw.Reviews.Nodes = []rawPRComment{rawComment(7, "hive[bot]", "Bot", "NONE", block, at.Add(time.Minute))} + raw.ReviewThreads.Nodes = []rawPRCommentThread{{ID: "T", Comments: struct { + Nodes []rawPRComment `json:"nodes"` + }{Nodes: []rawPRComment{rawComment(8, "hive[bot]", "Bot", "NONE", block, at.Add(-time.Minute))}}}} + got := HiveRuleReplies(raw, isHive) + var ids []string + for _, m := range got { + ids = append(ids, m.ID) + if !m.Hive || !strings.Contains(m.Body, FeedbackRulesMarker) { + t.Errorf("reply %+v", m) + } + } + if strings.Join(ids, ",") != "inline:8,review:7,conversation:3" { + t.Fatalf("ids = %v, want the hive's three block-bearing replies, oldest first", ids) + } +} diff --git a/src/pkg/knowledge/primer.go b/src/pkg/knowledge/primer.go index ff9ed0e4b0..8c45275281 100644 --- a/src/pkg/knowledge/primer.go +++ b/src/pkg/knowledge/primer.go @@ -22,6 +22,7 @@ type Context7Suggester func(ctx context.Context, keyword string) string type Primer struct { layers []layerClient fileStores []localStore + pinned []localStore graphStore *GraphStore config PrimerConfig logger *slog.Logger @@ -110,9 +111,20 @@ func (p *Primer) PrimeScoped(ctx context.Context, scope *TOCScope, filePaths []s func (p *Primer) PrimeForKick(ctx context.Context, scope *TOCScope, kc KickContext, keywords []string) *PrimedKnowledge { start := time.Now() + // An agent that opted into its own drafts (TOCScope.DraftAuthors, #17076) + // gets those on top of what the primer's lifecycle filter admits. + var draftAuthors []string + if scope != nil { + draftAuthors = scope.DraftAuthors + } + pinned := p.pinnedFacts(scope, kc, draftAuthors) + query := buildQuery(kc.Paths, keywords) if query == "" { - return &PrimedKnowledge{} + if len(pinned) == 0 { + return &PrimedKnowledge{} + } + return &PrimedKnowledge{Facts: pinned, MaxBytes: p.config.MaxBytes} } // Query each HTTP wiki layer; collect results tagged with their layer. @@ -152,12 +164,6 @@ func (p *Primer) PrimeForKick(ctx context.Context, scope *TOCScope, kc KickConte // Filter after the precedence merge so a higher-precedence layer that // deprecates a slug hides it rather than surfacing a lower layer's copy. - // An agent that opted into its own drafts (TOCScope.DraftAuthors, #17076) - // gets those on top of what the primer's lifecycle filter admits. - var draftAuthors []string - if scope != nil { - draftAuthors = scope.DraftAuthors - } merged := FilterByLifecycleFor(p.mergeWithPrecedence(allFacts), p.config.IncludeStates, draftAuthors) if scope != nil { merged = scope.Filter(merged) @@ -166,7 +172,10 @@ func (p *Primer) PrimeForKick(ctx context.Context, scope *TOCScope, kc KickConte reranked := p.rerankFisherRao(query, merged) prioritized := p.applyPriority(reranked) - if len(prioritized) > p.config.MaxFacts { + if len(pinned) > 0 { + prioritized = withPinned(pinned, prioritized, p.config.MaxFacts) + } + if len(prioritized) > p.config.MaxFacts && len(pinned) == 0 { prioritized = prioritized[:p.config.MaxFacts] } diff --git a/src/pkg/knowledge/primer_pinned.go b/src/pkg/knowledge/primer_pinned.go new file mode 100644 index 0000000000..f538e38a47 --- /dev/null +++ b/src/pkg/knowledge/primer_pinned.go @@ -0,0 +1,89 @@ +package knowledge + +import ( + "sort" + "strings" +) + +const ( + // PinTag marks a vault entry the primer carries on every kick whose + // context its scope admits, whatever the kick's keywords are. Reviewer + // rules captured from review feedback (hivecommons/hive#19046) use it: a + // rule a reviewer stated for an agent's future work must reach that + // agent's next run, not only the runs whose issue titles happen to share + // words with the rule. + PinTag = "pin" + // PinnedFactType is the section the primer renders pinned entries under. + // It is its own type so the section is rendered first and the byte cap + // reaches the pinned entries before it reaches ranked ones. + PinnedFactType FactType = "reviewer-rule" + // maxPinnedFacts bounds how many pinned entries one kick carries, so a + // large rule set cannot crowd out ranked knowledge. + maxPinnedFacts = 20 +) + +// AddPinnedFileStore registers a vault whose entries tagged PinTag are always +// primed when their scope admits the kick (FactScope.Admits; an entry without +// a scope admits every kick). The usual lifecycle filter and the agent's +// knowledge scope still apply, so a draft, deprecated or out-of-scope entry is +// never pinned. Like AddFileStore it is not safe for concurrent use with Prime. +func (p *Primer) AddPinnedFileStore(name string, store *FileStore, layer LayerType) { + p.pinned = append(p.pinned, localStore{layerType: layer, store: store, name: name}) + p.logger.Info("primer: added pinned file store", "name", name, "layer", layer) +} + +// HasPinned reports whether any pinned store is registered. +func (p *Primer) HasPinned() bool { return p != nil && len(p.pinned) > 0 } + +// pinnedFacts returns the pinned entries a kick must carry, oldest slug first +// so the result is stable. Bodies are read in full. +func (p *Primer) pinnedFacts(scope *TOCScope, kc KickContext, draftAuthors []string) []Fact { + var out []Fact + for _, ls := range p.pinned { + for _, f := range ls.store.ListPages(PinTag) { + full, err := ls.store.ReadPage(f.Slug) + if err != nil || full == nil { + continue + } + full.Layer = ls.layerType + full.Type = PinnedFactType + out = append(out, *full) + } + } + if len(out) == 0 { + return nil + } + out = FilterByLifecycleFor(out, p.config.IncludeStates, draftAuthors) + if scope != nil { + out = scope.Filter(out) + } + out, _ = FilterByKickContext(out, kc) + sort.Slice(out, func(i, j int) bool { return strings.Compare(out[i].Slug, out[j].Slug) < 0 }) + if len(out) > maxPinnedFacts { + out = out[:maxPinnedFacts] + } + return out +} + +// withPinned puts pinned first and drops any ranked copy of the same slug, +// then trims the ranked remainder so the total stays within maxFacts. Pinned +// entries are never trimmed. +func withPinned(pinned, ranked []Fact, maxFacts int) []Fact { + if len(pinned) == 0 { + return ranked + } + have := make(map[string]bool, len(pinned)) + for _, f := range pinned { + have[f.Slug] = true + } + room := maxFacts - len(pinned) + out := append([]Fact(nil), pinned...) + for _, f := range ranked { + if have[f.Slug] || room <= 0 { + continue + } + out = append(out, f) + room-- + } + return out +} diff --git a/src/pkg/knowledge/primer_pinned_test.go b/src/pkg/knowledge/primer_pinned_test.go new file mode 100644 index 0000000000..103bacae59 --- /dev/null +++ b/src/pkg/knowledge/primer_pinned_test.go @@ -0,0 +1,177 @@ +package knowledge + +import ( + "context" + "fmt" + "sort" + "strings" + "testing" +) + +func pinnedVault(t *testing.T) (string, *FileStore) { + t.Helper() + dir := t.TempDir() + long := "The maintainers file is the source of truth for ownership. " + strings.Repeat("Cite it in every assessment. ", 20) + writeLifecycleFact(t, dir, "rule-r-1", "title: R-1 quality rule\ntype: reviewer-rule\ntags: [review-rule, pin]\nscope:\n lanes: [quality]\n", long) + writeLifecycleFact(t, dir, "rule-r-2", "title: R-2 reviewer rule\ntype: reviewer-rule\ntags: [review-rule, pin]\nscope:\n lanes: [reviewer]\n", "Reviewer rule body.") + writeLifecycleFact(t, dir, "rule-r-3", "title: R-3 global rule\ntype: reviewer-rule\ntags: [review-rule, pin]\n", "Global rule body.") + writeLifecycleFact(t, dir, "rule-r-4", "title: R-4 queued rule\ntype: reviewer-rule\nstate: draft\ntags: [review-rule, pin]\nscope:\n lanes: [quality]\n", "Queued rule body.") + writeLifecycleFact(t, dir, "rule-r-5", "title: R-5 retracted rule\ntype: reviewer-rule\nstate: deprecated\ntags: [review-rule, pin]\nscope:\n lanes: [quality]\n", "Retracted rule body.") + writeLifecycleFact(t, dir, "rule-r-6", "title: R-6 repo rule\ntype: reviewer-rule\ntags: [review-rule, repo:cncf/techdocs]\nscope:\n lanes: [quality]\n", "Repo rule body about maintainers") + store, err := NewFileStore(dir, "review-rules", fileStoreTestLogger()) + if err != nil { + t.Fatal(err) + } + return dir, store +} + +func slugsOf(facts []Fact) string { + out := make([]string, 0, len(facts)) + for _, f := range facts { + out = append(out, f.Slug) + } + sort.Strings(out) + return strings.Join(out, ",") +} + +func TestPinnedFactsReachTheKickWhateverItsKeywords(t *testing.T) { + _, store := pinnedVault(t) + p := NewPrimer(nil, PrimerConfig{MaxFacts: 25}, fileStoreTestLogger()) + if p.HasPinned() { + t.Fatal("a fresh primer has nothing pinned") + } + p.AddPinnedFileStore("review-rules", store, LayerProject) + if !p.HasPinned() { + t.Fatal("HasPinned after AddPinnedFileStore") + } + tests := []struct { + name string + kc KickContext + keywords []string + want string + }{ + {"quality lane, no keywords", KickContext{Lane: "quality"}, nil, "rule-r-1,rule-r-3"}, + {"quality lane, unrelated keywords", KickContext{Lane: "quality"}, []string{"kubernetes"}, "rule-r-1,rule-r-3"}, + {"reviewer lane", KickContext{Lane: "reviewer"}, nil, "rule-r-2,rule-r-3"}, + {"other lane gets only the global rule", KickContext{Lane: "developer"}, nil, "rule-r-3"}, + {"no context gets only the global rule", KickContext{}, []string{"x"}, "rule-r-3"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := p.PrimeForKick(context.Background(), nil, tc.kc, tc.keywords) + if slugsOf(got.Facts) != tc.want { + t.Fatalf("pinned = %s, want %s", slugsOf(got.Facts), tc.want) + } + for _, f := range got.Facts { + if f.Type != PinnedFactType || f.Layer != LayerProject { + t.Fatalf("pinned fact %s has type %q layer %q", f.Slug, f.Type, f.Layer) + } + } + }) + } +} + +func TestPinnedFactsAreRenderedWholeAndFirst(t *testing.T) { + _, store := pinnedVault(t) + p := NewPrimer(nil, PrimerConfig{MaxFacts: 25}, fileStoreTestLogger()) + p.AddPinnedFileStore("review-rules", store, LayerProject) + primed := p.PrimeForKick(context.Background(), nil, KickContext{Lane: "quality"}, nil) + out, omitted := primed.Render() + if omitted.Any() { + t.Fatalf("omitted = %+v", omitted) + } + if !strings.Contains(out, "## reviewer-rule") || !strings.Contains(out, "The maintainers file is the source of truth for ownership.") { + t.Fatalf("render:\n%s", out) + } + if !strings.Contains(out, strings.Repeat("Cite it in every assessment. ", 20)[:200]) { + t.Fatal("the pinned body must be read in full, not as a list snippet") + } +} + +func TestPinnedFactsHonourTheAgentScope(t *testing.T) { + _, store := pinnedVault(t) + p := NewPrimer(nil, PrimerConfig{MaxFacts: 25}, fileStoreTestLogger()) + p.AddPinnedFileStore("review-rules", store, LayerProject) + narrow := AgentScope([]string{"org"}, nil, nil, nil, nil) + if got := p.PrimeForKick(context.Background(), &narrow, KickContext{Lane: "quality"}, nil); len(got.Facts) != 0 { + t.Fatalf("an agent scoped to another layer must not receive pinned rules: %s", slugsOf(got.Facts)) + } + open := AgentScope([]string{"project"}, nil, nil, nil, nil) + if got := p.PrimeForKick(context.Background(), &open, KickContext{Lane: "quality"}, nil); slugsOf(got.Facts) != "rule-r-1,rule-r-3" { + t.Fatalf("pinned = %s", slugsOf(got.Facts)) + } +} + +func TestPinnedFactsMergeWithRankedFacts(t *testing.T) { + dir, store := pinnedVault(t) + p := NewPrimer(nil, PrimerConfig{MaxFacts: 25}, fileStoreTestLogger()) + p.AddFileStore("review-rules", store, LayerProject) + p.AddPinnedFileStore("review-rules", store, LayerProject) + got := p.PrimeForKick(context.Background(), nil, KickContext{Lane: "quality"}, []string{"maintainers"}) + seen := map[string]int{} + for _, f := range got.Facts { + seen[f.Slug]++ + } + if seen["rule-r-1"] != 1 || seen["rule-r-3"] != 1 { + t.Fatalf("a pinned fact that ranked too appears once: %v (dir %s)", seen, dir) + } + if got.Facts[0].Type != PinnedFactType { + t.Fatalf("pinned facts come first: %s", slugsOf(got.Facts[:1])) + } + // A ranked, unpinned entry still arrives by relevance. + if seen["rule-r-6"] != 1 { + t.Fatalf("the unpinned repo rule is retrievable by keyword: %v", seen) + } +} + +func TestPinnedFactsAreBoundedAndNeverTrimmed(t *testing.T) { + dir := t.TempDir() + for i := 0; i < maxPinnedFacts+5; i++ { + writeLifecycleFact(t, dir, fmt.Sprintf("rule-%02d", i), fmt.Sprintf("title: Rule %d\ntype: reviewer-rule\ntags: [pin]\n", i), "Body.") + } + store, err := NewFileStore(dir, "review-rules", fileStoreTestLogger()) + if err != nil { + t.Fatal(err) + } + p := NewPrimer(nil, PrimerConfig{MaxFacts: 3}, fileStoreTestLogger()) + p.AddPinnedFileStore("review-rules", store, LayerProject) + if got := p.PrimeForKick(context.Background(), nil, KickContext{Lane: "quality"}, nil); len(got.Facts) != maxPinnedFacts { + t.Fatalf("pinned = %d, want the %d cap", len(got.Facts), maxPinnedFacts) + } + if got := p.PrimeForKick(context.Background(), nil, KickContext{Lane: "quality"}, []string{"body"}); len(got.Facts) != maxPinnedFacts { + t.Fatalf("with keywords pinned = %d, want %d", len(got.Facts), maxPinnedFacts) + } +} + +func TestWithPinned(t *testing.T) { + pin := []Fact{{Slug: "p1"}, {Slug: "p2"}} + ranked := []Fact{{Slug: "a"}, {Slug: "p1"}, {Slug: "b"}, {Slug: "c"}} + tests := []struct { + name string + max int + want string + }{ + {"room for two ranked", 4, "p1,p2,a,b"}, + {"no room", 2, "p1,p2"}, + {"max below pinned count keeps pinned", 1, "p1,p2"}, + {"ample", 10, "p1,p2,a,b,c"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var slugs []string + for _, f := range withPinned(pin, ranked, tc.max) { + slugs = append(slugs, f.Slug) + } + if strings.Join(slugs, ",") != tc.want { + t.Fatalf("got %v, want %s", slugs, tc.want) + } + }) + } + if got := withPinned(nil, ranked, 2); len(got) != len(ranked) { + t.Fatal("with nothing pinned the ranked list passes through for the caller to trim") + } + var none *Primer + if none.HasPinned() { + t.Fatal("nil primer has nothing pinned") + } +} diff --git a/src/pkg/reviewfeedback/reviewfeedback.go b/src/pkg/reviewfeedback/reviewfeedback.go index b6400969e4..1cbbf96c6f 100644 --- a/src/pkg/reviewfeedback/reviewfeedback.go +++ b/src/pkg/reviewfeedback/reviewfeedback.go @@ -323,6 +323,31 @@ func (s *Store) saveLocked(now time.Time) error { // follow anything inside them. Comments in one review thread are shown as the // whole thread in order, so the agent applies the reviewer's latest position. func BuildBrief(repo string, number int, headRef, writingGuide string, items []github.FeedbackItem) string { + return BuildBriefWithRules(repo, number, headRef, writingGuide, items, nil) +} + +// RecordedRule is a reviewer-stated rule the hive already captured into its +// knowledge store before the kick (pkg/reviewrules), shown to the agent so it +// refers to it instead of recording it again. +type RecordedRule struct { + // ID is the rule id, "R-3". + ID string + // Text is the rule as recorded. + Text string + // Applies says where the rule applies, in words. + Applies string + // Queued is true when the reviewer is not trusted to state rules on their + // own authority and a person must confirm the rule first. + Queued bool + // SourceURL is the comment the rule came from. + SourceURL string +} + +// BuildBriefWithRules is BuildBrief plus the rules the hive already captured +// from this burst. With rules, item 6 tells the agent they are recorded, to +// name them in its reply, and how to sharpen one with a structured block; +// without, item 6 keeps the best-effort instruction to record a stated rule. +func BuildBriefWithRules(repo string, number int, headRef, writingGuide string, items []github.FeedbackItem, rules []RecordedRule) string { var b strings.Builder fmt.Fprintf(&b, "REVIEW FEEDBACK on your PR %s#%d needs a response before anything new\n\n", repo, number) b.WriteString("A person left feedback on a PR you opened. Revisit it now, whether or not the PR is a draft, carries ready-for-review, or sits in a repo you do not own. A hold label only blocks the merge; it never means do not respond. This is one burst: every comment below is new, answer them together in one revision.\n\n") @@ -334,7 +359,22 @@ func BuildBrief(repo string, number int, headRef, writingGuide string, items []g b.WriteString(" 3. If the reviewer is right: fix it on the SAME branch (gh pr checkout, edit, git commit -s, git push). No replacement PR.\n") b.WriteString(" 4. If the reviewer is wrong or it is a judgment call: reply politely with the evidence. Never argue to defend a score or to please the reviewer; accuracy is the only goal.\n") b.WriteString(" 5. Reply inside each thread (gh api repos/OWNER/REPO/pulls/N/comments/ID/replies), or on the PR for review and conversation comments. Each reply says what you verified, what you changed and in which commit, and what you left unchanged and why.\n") - b.WriteString(" 6. Rules, not only corrections. If a comment states a rule or a source of truth for future assessments (for example which file is authoritative for ownership), record it durably: write a knowledge fact in the hive's own knowledge store with `hivectl knowledge create --file fact.yaml` (title, body, tags, and the PR comment URL as the source), then say so in your reply. If the right home for the rule is another project's repo (for example its criteria files), do NOT write there: propose the exact text in your reply and say a person needs to place it. If hivectl cannot write, say so in the reply instead of dropping the rule.\n") + if len(rules) == 0 { + b.WriteString(" 6. Rules, not only corrections. If a comment states a rule or a source of truth for future assessments (for example which file is authoritative for ownership), record it durably: write a knowledge fact in the hive's own knowledge store with `hivectl knowledge create --file fact.yaml` (title, body, tags, and the PR comment URL as the source), then say so in your reply. If the right home for the rule is another project's repo (for example its criteria files), do NOT write there: propose the exact text in your reply and say a person needs to place it. If hivectl cannot write, say so in the reply instead of dropping the rule.\n") + } else { + b.WriteString(" 6. Rules, not only corrections. The hive already recorded the rules below from this burst in its own knowledge store, so do NOT run hivectl knowledge create for them. Name each rule id in your reply so the reviewer sees it was captured. A rule applies to future work as well as this PR: apply it here too. If the wording or scope should be sharper, or the right home is another project's repo (do NOT write there; the hive posts the proposed text for a person), end your reply with one hidden line `` (JSON, write > inside text as \\u003e). Cite as evidence_url only a comment URL shown below. If a comment below states a rule that is not listed, add it the same way.\n") + for _, r := range rules { + state := "recorded" + if r.Queued { + state = "queued for a person to confirm" + } + fmt.Fprintf(&b, " - %s (%s; applies to %s): %s", r.ID, state, r.Applies, truncate(r.Text, threadMessageRunes)) + if r.SourceURL != "" { + fmt.Fprintf(&b, " [source %s]", r.SourceURL) + } + b.WriteString("\n") + } + } fmt.Fprintf(&b, " 7. End every reply set with one hidden line so these comments are not routed again: %s\n", github.FeedbackHandledMarker(feedbackIDs(items))) b.WriteString("If you cannot push (no write access to the branch), say so in a PR comment that names what you would change, and stop.\n") b.WriteString("After replying, STOP. Do not poll CI or wait for the reviewer. More feedback that arrives while you work is sent as a separate follow-up.\n\n") diff --git a/src/pkg/reviewrules/apply_test.go b/src/pkg/reviewrules/apply_test.go new file mode 100644 index 0000000000..90679dba4c --- /dev/null +++ b/src/pkg/reviewrules/apply_test.go @@ -0,0 +1,328 @@ +package reviewrules + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func managedOnly(repos ...string) func(string) bool { + return func(r string) bool { + for _, m := range repos { + if m == r { + return true + } + } + return false + } +} + +func replyBlock(rules string) string { + return "Fixed in abc123.\n\n" +} + +func entry(text, scope, applies, url string) string { + return `{"text":"` + text + `","scope":"` + scope + `","applies_to":"` + applies + `","evidence_url":"` + url + `"}` +} + +func seeded(t *testing.T, assoc string) (*Store, string, string, Comment) { + t.Helper() + s, dir, vault := openStore(t, Config{}) + c := cmt("inline:1001", assoc, maintainersComment) + if err := s.Remember(c, t0); err != nil { + t.Fatal(err) + } + return s, dir, vault, c +} + +func TestApplyReplyRefinesACapturedRule(t *testing.T) { + s, dir, vault, c := seeded(t, "OWNER") + s.CaptureComment(c, t0) + body := replyBlock(entry("The maintainers file is the source of truth for ownership in every assessment.", "workflow", "cncf-techdocs-assessment", c.URL)) + got := s.ApplyReply("reply-1", body, managedOnly(), t0.Add(time.Hour)) + if len(got) != 1 || got[0].Outcome != OutcomeRefined { + t.Fatalf("applied = %+v", got) + } + r := s.Get("R-1") + if r.Scope != ScopeWorkflow || r.AppliesTo != "cncf-techdocs-assessment" || !strings.Contains(r.Text, "every assessment") || r.Confidence != confidenceAgent { + t.Fatalf("rule = %+v", r) + } + if !strings.Contains(readFile(t, filepath.Join(vault, "rule-r-1.md")), "scope:\n lanes: [cncf-techdocs-assessment]") { + t.Fatal("the workflow scope must reach the entry") + } + // The same reply applies once, across a reload too. + if again := s.ApplyReply("reply-1", body, nil, t0); again != nil { + t.Fatalf("a reply is applied once: %+v", again) + } + reopened, _ := Open(dir, vault, Config{}) + if again := reopened.ApplyReply("reply-1", body, nil, t0); again != nil { + t.Fatalf("a reply is applied once across a reload: %+v", again) + } +} + +func TestApplyReplyScopeDefaults(t *testing.T) { + tests := []struct { + name string + scope string + appliesTo string + wantScope Scope + wantApply string + }{ + {"scope omitted is the authoring agent", "", "", ScopeAgent, "quality"}, + {"agent without a name", "agent", "", ScopeAgent, "quality"}, + {"agent with a name", "agent", "reviewer", ScopeAgent, "reviewer"}, + {"global drops the name", "global", "ignored", ScopeGlobal, ""}, + {"repo keeps the slug", "repo", "cncf/techdocs", ScopeRepo, "cncf/techdocs"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s, _, _, c := seeded(t, "OWNER") + s.CaptureComment(c, t0) + got := s.ApplyReply("r", replyBlock(entry("Use the maintainers file as the ownership source of truth.", tc.scope, tc.appliesTo, c.URL)), managedOnly("cncf/techdocs"), t0) + if len(got) != 1 || got[0].Outcome != OutcomeRefined { + t.Fatalf("applied = %+v", got) + } + if r := s.Get("R-1"); r.Scope != tc.wantScope || r.AppliesTo != tc.wantApply { + t.Fatalf("scope=%s applies_to=%q, want %s %q", r.Scope, r.AppliesTo, tc.wantScope, tc.wantApply) + } + }) + } +} + +func TestApplyReplyRejectsWhatItCannotTie(t *testing.T) { + s, _, _, c := seeded(t, "OWNER") + s.CaptureComment(c, t0) + tests := []struct { + name string + body string + reason string + }{ + {"evidence the hive never saw", replyBlock(entry("t", "", "", "https://github.com/cncf/techdocs/pull/424#discussion_r999")), "the hive saw"}, + {"invalid entry", replyBlock(entry("", "", "", c.URL)), "text is empty"}, + {"not json", "x ", "valid JSON"}, + } + for i, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := s.ApplyReply("rej-"+string(rune('a'+i)), tc.body, nil, t0) + if len(got) == 0 || got[0].Outcome != OutcomeRejected || !strings.Contains(got[0].Reason, tc.reason) { + t.Fatalf("applied = %+v, want a rejection mentioning %q", got, tc.reason) + } + }) + } + if len(s.List()) != 1 || s.Get("R-1").Text != ".project/maintainers.yaml is the source of truth for ownership and maintainership" { + t.Fatal("a rejected block must change nothing") + } + // A reply with no block is not recorded at all. + if got := s.ApplyReply("plain", "Fixed, thanks.", nil, t0); got != nil || len(s.st.Replies) != 3 { + t.Fatalf("plain reply: applied=%v replies=%d", got, len(s.st.Replies)) + } +} + +func TestApplyReplyCapturesFromACommentTheClassifierMissed(t *testing.T) { + s, _, vault := openStore(t, Config{}) + missed := cmt("inline:2002", "MEMBER", "Agent readiness is scored from the repo's AGENTS.md alone.") + untrusted := cmt("inline:2003", "CONTRIBUTOR", "Weights come from the rubric page.") + for _, c := range []Comment{missed, untrusted} { + if err := s.Remember(c, t0); err != nil { + t.Fatal(err) + } + } + body := replyBlock(entry("Score agent readiness from the AGENTS.md file only.", "agent", "quality", missed.URL) + "," + + entry("Take weights from the rubric page.", "", "", untrusted.URL)) + got := s.ApplyReply("r", body, nil, t0) + if len(got) != 2 || got[0].Outcome != OutcomeCaptured || got[1].Outcome != OutcomeQueued { + t.Fatalf("applied = %+v", got) + } + if r := got[0].Rule; r.Provenance[0].Source != "agent" || r.Confidence != confidenceAgentFromSeen || r.Status != StatusActive { + t.Fatalf("rule = %+v", r) + } + if got[1].Rule.Status != StatusPending { + t.Fatal("trust comes from the cited comment's association, never from the agent") + } + if !strings.Contains(readFile(t, filepath.Join(vault, "rule-r-2.md")), "state: draft") { + t.Fatal("a queued agent rule is a draft entry") + } +} + +func TestApplyReplyForeignHome(t *testing.T) { + s, _, _, c := seeded(t, "OWNER") + s.CaptureComment(c, t0) + body := replyBlock(entry("Projects list owners in their maintainers file.", "repo", "cncf/techdocs", c.URL)) + if got := s.ApplyReply("r1", body, managedOnly("hivecommons/hive"), t0); len(got) != 1 || got[0].Outcome != OutcomeRefined { + t.Fatalf("applied = %+v", got) + } + props := s.HomeProposals() + if len(props) != 1 || props[0].ForeignHome != "cncf/techdocs" || props[0].Scope != ScopeRepo { + t.Fatalf("proposals = %+v", props) + } + if err := s.MarkHomeNoted("R-1"); err != nil { + t.Fatal(err) + } + // The same destination does not re-open the proposal; a new one does. + s.ApplyReply("r2", replyBlock(entry("Projects list owners in their maintainers file, always.", "repo", "cncf/techdocs", c.URL)), managedOnly(), t0) + if len(s.HomeProposals()) != 0 { + t.Fatal("an unchanged destination is not proposed twice") + } + s.ApplyReply("r3", replyBlock(entry("Projects list owners in their maintainers file, always.", "repo", "other/project", c.URL)), managedOnly(), t0) + if p := s.HomeProposals(); len(p) != 1 || p[0].ForeignHome != "other/project" { + t.Fatalf("a new destination is proposed once: %+v", p) + } + // A managed destination is the hive's own: no proposal. + s2, _, _, c2 := seeded(t, "OWNER") + s2.CaptureComment(c2, t0) + s2.ApplyReply("r", replyBlock(entry("t", "repo", "hivecommons/hive", c2.URL)), managedOnly("hivecommons/hive"), t0) + if len(s2.HomeProposals()) != 0 { + t.Fatal("a managed repo needs no proposal") + } +} + +func TestApplyReplyMergesARefinedRuleIntoItsTwin(t *testing.T) { + s, _, vault := openStore(t, Config{}) + c1 := cmt("inline:1", "OWNER", maintainersComment) + c2 := cmt("inline:2", "OWNER", "Going forward, cite the scorecard with a commit-pinned link.") + for _, c := range []Comment{c1, c2} { + _ = s.Remember(c, t0) + s.CaptureComment(c, t0) + } + got := s.ApplyReply("r", replyBlock(entry("The .project/maintainers.yaml is the source of truth for ownership and maintainership.", "", "", c2.URL)), nil, t0.Add(time.Hour)) + if len(got) != 1 || got[0].Outcome != OutcomeMerged || got[0].Rule.ID != "R-1" || len(got[0].Rule.Provenance) != 2 { + t.Fatalf("applied = %+v", got) + } + r2 := s.Get("R-2") + if r2.Status != StatusMerged || r2.MergedInto != "R-1" { + t.Fatalf("R-2 = %+v", r2) + } + entry2 := readFile(t, filepath.Join(vault, "rule-r-2.md")) + if !strings.Contains(entry2, "state: superseded\n") || !strings.Contains(entry2, "superseded_by: rule-r-1\n") { + t.Fatalf("a merged rule's entry must be superseded by the survivor:\n%s", entry2) + } + if again := s.CaptureComment(c2, t0); again.Outcome != OutcomeDuplicate || again.Rule.ID != "R-1" { + t.Fatalf("a merged rule's comment resolves to the survivor: %+v", again) + } + // Refining the older rule to the newer one's wording keeps the older id. + s3, _, _ := openStore(t, Config{}) + for _, c := range []Comment{c1, c2} { + _ = s3.Remember(c, t0) + s3.CaptureComment(c, t0) + } + out := s3.ApplyReply("r", replyBlock(entry("Going forward, cite the scorecard with a commit-pinned link.", "", "", c1.URL)), nil, t0) + if len(out) != 1 || out[0].Outcome != OutcomeMerged || out[0].Rule.ID != "R-1" || s3.Get("R-2").MergedInto != "R-1" { + t.Fatalf("older id must survive: %+v", out) + } +} + +func TestApplyReplyRefinesOnlyLiveRules(t *testing.T) { + s, _, _, c := seeded(t, "OWNER") + s.CaptureComment(c, t0) + if err := s.Retract("R-1", "operator", t0); err != nil { + t.Fatal(err) + } + got := s.ApplyReply("r", replyBlock(entry("Use the maintainers file.", "", "", c.URL)), nil, t0) + if len(got) != 1 || got[0].Outcome != OutcomeRejected || !strings.Contains(got[0].Reason, "no longer live") { + t.Fatalf("applied = %+v", got) + } +} + +func TestApplyReplyRefinementOfAQueuedRuleKeepsItQueued(t *testing.T) { + s, _, _, c := seeded(t, "CONTRIBUTOR") + s.CaptureComment(c, t0) + got := s.ApplyReply("r", replyBlock(entry("Use the maintainers file as the ownership source of truth.", "", "", c.URL)), nil, t0) + if len(got) != 1 || got[0].Outcome != OutcomeRefined || got[0].Rule.Status != StatusPending || got[0].Rule.Confidence != confidencePending { + t.Fatalf("applied = %+v", got) + } +} + +func TestMergeActivatesAQueuedSurvivor(t *testing.T) { + s, _, _ := openStore(t, Config{}) + queued := cmt("inline:1", "CONTRIBUTOR", maintainersComment) + trusted := cmt("inline:2", "OWNER", "Going forward, cite the scorecard with a commit-pinned link.") + for _, c := range []Comment{queued, trusted} { + _ = s.Remember(c, t0) + s.CaptureComment(c, t0) + } + got := s.ApplyReply("r", replyBlock(entry("The .project/maintainers.yaml is the source of truth for ownership and maintainership.", "", "", trusted.URL)), nil, t0) + if len(got) != 1 || got[0].Outcome != OutcomeMerged || got[0].Rule.Status != StatusActive { + t.Fatalf("a trusted twin must activate the queued survivor: %+v", got) + } +} + +func TestApplyReplyPrunesRememberedReplies(t *testing.T) { + s, _, _, c := seeded(t, "OWNER") + s.CaptureComment(c, t0) + for i := 0; i < maxRepliesKept+3; i++ { + id := "reply-" + strings.Repeat("x", i%7) + string(rune('a'+i%26)) + strings.Repeat("y", i/26) + s.ApplyReply(id, "", nil, t0.Add(time.Duration(i)*time.Second)) + } + if len(s.st.Replies) > maxRepliesKept { + t.Fatalf("replies = %d, want at most %d", len(s.st.Replies), maxRepliesKept) + } +} + +func TestVaultEntryShapes(t *testing.T) { + s, _, vault := openStore(t, Config{}) + repoRule := cmt("inline:1", "OWNER", "Going forward, list owners in the maintainers file.") + s.CaptureComment(repoRule, t0) + _ = s.Remember(repoRule, t0) + s.ApplyReply("r", replyBlock(entry("List owners in the maintainers file.", "repo", "cncf/techdocs", repoRule.URL)), managedOnly("cncf/techdocs"), t0) + b := readFile(t, filepath.Join(vault, "rule-r-1.md")) + if strings.Contains(b, ", pin") || !strings.Contains(b, "repo:cncf/techdocs") || !strings.Contains(b, "scope:\n lanes: [quality]") { + t.Fatalf("a repo rule is not pinned, carries its repo tag and stays in its lane:\n%s", b) + } + global := cmt("inline:2", "OWNER", "From now on always run the linter before pushing.") + _ = s.Remember(global, t0) + s.CaptureComment(global, t0) + s.ApplyReply("g", replyBlock(entry("Always run the linter before pushing.", "global", "", global.URL)), nil, t0) + g := readFile(t, filepath.Join(vault, "rule-r-2.md")) + if !strings.Contains(g, ", pin") || strings.Contains(g, "scope:") { + t.Fatalf("a global rule is pinned and unscoped:\n%s", g) + } + if (&Rule{Scope: ScopeRepo}).lanes() != nil { + t.Fatal("a repo rule without a lane has no lane scope") + } +} + +func TestDescribeAndHelpers(t *testing.T) { + tests := []struct { + r Rule + want string + }{ + {Rule{Scope: ScopeAgent, AppliesTo: "quality"}, "the `quality` agent's future work"}, + {Rule{Scope: ScopeWorkflow, AppliesTo: "wf"}, "the `wf` workflow"}, + {Rule{Scope: ScopeRepo, AppliesTo: "o/r"}, "work on `o/r`"}, + {Rule{Scope: ScopeGlobal}, "all agents' work"}, + } + for _, tc := range tests { + if got := tc.r.Describe(); got != tc.want { + t.Errorf("Describe = %q, want %q", got, tc.want) + } + } + if got := tagValue("a b/c:d!"); got != "a-b/c:d" { + t.Errorf("tagValue = %q", got) + } + if got := firstRunes("abcdef", 3); got != "abc..." { + t.Errorf("firstRunes = %q", got) + } + if got := firstRunes("ab", 3); got != "ab" { + t.Errorf("firstRunes = %q", got) + } + if got := errText(nil); got != "" { + t.Errorf("errText = %q", got) + } +} + +func TestEntryWriteFailureIsReported(t *testing.T) { + blocker := filepath.Join(t.TempDir(), "file") + if err := os.WriteFile(blocker, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + s, err := Open(t.TempDir(), filepath.Join(blocker, "vault"), Config{}) + if err != nil { + t.Fatal(err) + } + res := s.CaptureComment(cmt("inline:1", "OWNER", maintainersComment), t0) + if res.Outcome != OutcomeRefused || res.Err == nil || len(s.List()) != 0 { + t.Fatalf("result = %+v, want a refusal when the vault cannot be written", res) + } +} diff --git a/src/pkg/reviewrules/block.go b/src/pkg/reviewrules/block.go new file mode 100644 index 0000000000..302ffceebd --- /dev/null +++ b/src/pkg/reviewrules/block.go @@ -0,0 +1,124 @@ +package reviewrules + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" + "unicode/utf8" +) + +// BlockMarker names the hidden block a kicked agent may end its reply with. +// The block is the agent's own structured statement of the rules the reviewer +// stated; the hive validates it field by field and trusts none of it beyond +// what it can tie to a review comment it saw. +const BlockMarker = "hive:review-rules" + +// MaxBlockRules bounds how many rules one reply may carry. +const MaxBlockRules = 10 + +const ( + maxEvidenceURLLen = 300 + maxAppliesToLen = 100 +) + +var blockRe = regexp.MustCompile(`(?s)`) + +// evidenceURLRe accepts only a GitHub pull request or issue URL, optionally +// with a comment anchor. +var evidenceURLRe = regexp.MustCompile(`^https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/(?:pull|issues)/[0-9]+(?:#[A-Za-z0-9_-]+)?$`) + +var appliesToRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]*$`) + +var repoSlugRe = regexp.MustCompile(`^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$`) + +// Scope says where a rule applies. +type Scope string + +const ( + // ScopeAgent applies to one agent's work, across repos. + ScopeAgent Scope = "agent" + // ScopeWorkflow applies to one named workflow's runs. + ScopeWorkflow Scope = "workflow" + // ScopeRepo applies to one repository. + ScopeRepo Scope = "repo" + // ScopeGlobal applies to every agent. + ScopeGlobal Scope = "global" +) + +// BlockRule is one entry of the agent's structured block, as validated. +type BlockRule struct { + Text string `json:"text"` + Scope string `json:"scope"` + EvidenceURL string `json:"evidence_url"` + AppliesTo string `json:"applies_to"` +} + +type rawBlock struct { + Rules []BlockRule `json:"rules"` +} + +// ParseBlocks reads every hive:review-rules block in a reply body. Entries that +// fail validation are returned as errors, one per entry, and never as rules: +// one bad entry does not discard its valid siblings. The total is capped at +// MaxBlockRules; a reply with more has the excess reported as an error. +func ParseBlocks(body string) ([]BlockRule, []error) { + var rules []BlockRule + var errs []error + for _, m := range blockRe.FindAllStringSubmatch(body, -1) { + dec := json.NewDecoder(strings.NewReader(m[1])) + dec.DisallowUnknownFields() + var rb rawBlock + if err := dec.Decode(&rb); err != nil { + errs = append(errs, fmt.Errorf("block is not valid JSON of the expected shape: %w", err)) + continue + } + for i, r := range rb.Rules { + if len(rules) >= MaxBlockRules { + errs = append(errs, fmt.Errorf("more than %d rules in one reply; the rest are ignored", MaxBlockRules)) + return rules, errs + } + v, err := ValidateBlockRule(r) + if err != nil { + errs = append(errs, fmt.Errorf("rule %d: %w", i+1, err)) + continue + } + rules = append(rules, v) + } + } + return rules, errs +} + +// ValidateBlockRule checks one entry and returns it normalised. +func ValidateBlockRule(r BlockRule) (BlockRule, error) { + r.Text = NormalizeText(r.Text) + switch { + case r.Text == "": + return r, fmt.Errorf("text is empty") + case utf8.RuneCountInString(r.Text) > MaxRuleRunes: + return r, fmt.Errorf("text is over %d characters", MaxRuleRunes) + } + r.Scope = strings.ToLower(strings.TrimSpace(r.Scope)) + r.AppliesTo = strings.TrimSpace(r.AppliesTo) + r.EvidenceURL = strings.TrimSpace(r.EvidenceURL) + if len(r.EvidenceURL) > maxEvidenceURLLen || !evidenceURLRe.MatchString(r.EvidenceURL) { + return r, fmt.Errorf("evidence_url must be a GitHub pull request or issue comment URL") + } + if r.AppliesTo != "" && (len(r.AppliesTo) > maxAppliesToLen || !appliesToRe.MatchString(r.AppliesTo)) { + return r, fmt.Errorf("applies_to has characters a name cannot have") + } + switch Scope(r.Scope) { + case "", ScopeAgent, ScopeGlobal: + case ScopeWorkflow: + if r.AppliesTo == "" { + return r, fmt.Errorf("scope workflow needs applies_to, the workflow name") + } + case ScopeRepo: + if !repoSlugRe.MatchString(r.AppliesTo) { + return r, fmt.Errorf("scope repo needs applies_to as owner/name") + } + default: + return r, fmt.Errorf("scope must be one of agent, workflow, repo, global") + } + return r, nil +} diff --git a/src/pkg/reviewrules/block_test.go b/src/pkg/reviewrules/block_test.go new file mode 100644 index 0000000000..aad1880004 --- /dev/null +++ b/src/pkg/reviewrules/block_test.go @@ -0,0 +1,76 @@ +package reviewrules + +import ( + "strings" + "testing" +) + +const goodURL = "https://github.com/cncf/techdocs/pull/424#discussion_r1001" + +func block(json string) string { return "Fixed.\n\n" } + +func TestParseBlocks(t *testing.T) { + tests := []struct { + name string + body string + want int + wantErrs []string + }{ + {"no block", "just a reply", 0, nil}, + {"valid agent scope default", block(`{"rules":[{"text":"Use the maintainers file.","evidence_url":"` + goodURL + `"}]}`), 1, nil}, + {"valid workflow", block(`{"rules":[{"text":"t","scope":"workflow","applies_to":"cncf-techdocs-assessment","evidence_url":"` + goodURL + `"}]}`), 1, nil}, + {"valid repo", block(`{"rules":[{"text":"t","scope":"Repo","applies_to":"cncf/techdocs","evidence_url":"` + goodURL + `"}]}`), 1, nil}, + {"valid global", block(`{"rules":[{"text":"t","scope":"global","evidence_url":"` + goodURL + `"}]}`), 1, nil}, + {"malformed json", block(`{"rules":[`), 0, nil}, + {"unknown field", block(`{"rules":[{"text":"t","evidence_url":"` + goodURL + `","cmd":"rm -rf /"}]}`), 0, []string{"valid JSON"}}, + {"empty text", block(`{"rules":[{"text":" ","evidence_url":"` + goodURL + `"}]}`), 0, []string{"text is empty"}}, + {"long text", block(`{"rules":[{"text":"` + strings.Repeat("x", MaxRuleRunes+1) + `","evidence_url":"` + goodURL + `"}]}`), 0, []string{"over 600"}}, + {"evidence not github", block(`{"rules":[{"text":"t","evidence_url":"https://evil.example/pull/1"}]}`), 0, []string{"evidence_url"}}, + {"evidence missing", block(`{"rules":[{"text":"t"}]}`), 0, []string{"evidence_url"}}, + {"evidence http", block(`{"rules":[{"text":"t","evidence_url":"http://github.com/a/b/pull/1"}]}`), 0, []string{"evidence_url"}}, + {"bad scope", block(`{"rules":[{"text":"t","scope":"everything","evidence_url":"` + goodURL + `"}]}`), 0, []string{"scope must be"}}, + {"workflow without name", block(`{"rules":[{"text":"t","scope":"workflow","evidence_url":"` + goodURL + `"}]}`), 0, []string{"needs applies_to"}}, + {"repo without slug", block(`{"rules":[{"text":"t","scope":"repo","applies_to":"nope","evidence_url":"` + goodURL + `"}]}`), 0, []string{"owner/name"}}, + {"applies_to bad characters", block(`{"rules":[{"text":"t","applies_to":"a b; rm","evidence_url":"` + goodURL + `"}]}`), 0, []string{"applies_to"}}, + {"one bad one good", block(`{"rules":[{"text":"","evidence_url":"` + goodURL + `"},{"text":"ok","evidence_url":"` + goodURL + `"}]}`), 1, []string{"rule 1"}}, + {"two blocks", block(`{"rules":[{"text":"a","evidence_url":"`+goodURL+`"}]}`) + block(`{"rules":[{"text":"b","evidence_url":"`+goodURL+`"}]}`), 2, nil}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + rules, errs := ParseBlocks(tc.body) + if len(rules) != tc.want { + t.Fatalf("rules = %d, want %d (errs %v)", len(rules), tc.want, errs) + } + for _, w := range tc.wantErrs { + found := false + for _, e := range errs { + found = found || strings.Contains(e.Error(), w) + } + if !found { + t.Fatalf("errors %v do not mention %q", errs, w) + } + } + }) + } +} + +func TestParseBlocksCapsTheCount(t *testing.T) { + var items []string + for i := 0; i < MaxBlockRules+3; i++ { + items = append(items, `{"text":"r","evidence_url":"`+goodURL+`"}`) + } + rules, errs := ParseBlocks(block(`{"rules":[` + strings.Join(items, ",") + `]}`)) + if len(rules) != MaxBlockRules || len(errs) != 1 || !strings.Contains(errs[0].Error(), "more than") { + t.Fatalf("rules=%d errs=%v, want %d rules and one cap error", len(rules), errs, MaxBlockRules) + } +} + +func TestValidateBlockRuleNormalises(t *testing.T) { + got, err := ValidateBlockRule(BlockRule{Text: " a\n b ", Scope: " AGENT ", EvidenceURL: " " + goodURL + " ", AppliesTo: " quality "}) + if err != nil { + t.Fatal(err) + } + if got.Text != "a b c" || got.Scope != "agent" || got.AppliesTo != "quality" || got.EvidenceURL != goodURL { + t.Fatalf("normalised = %+v", got) + } +} diff --git a/src/pkg/reviewrules/classify.go b/src/pkg/reviewrules/classify.go new file mode 100644 index 0000000000..4747e7e226 --- /dev/null +++ b/src/pkg/reviewrules/classify.go @@ -0,0 +1,173 @@ +// Package reviewrules captures the rules and sources of truth that people +// state in review feedback on the hive's own agent PRs, so the hive applies +// them to future work and not only to the PR they were written on +// (hivecommons/hive#19046). +// +// The review-feedback sweep (pkg/reviewfeedback, cmd/hive) kicks the authoring +// agent to fix the PR in hand. A fix applies to one PR; a stated rule ("the +// project's maintainers file is the source of truth for ownership") applies to +// every later analysis. This package makes the second half deterministic: +// +// - Classify decides, from explicit phrase signals, whether a comment states +// a rule or is a one-off correction. +// - Store persists captured rules with their provenance, deduplicated by +// comment id and by near-identical text, as scoped, pinned entries of a +// local knowledge vault the kick primer loads for the matching agent. +// - ParseBlocks reads the structured block the kicked agent may end its +// reply with to sharpen a rule's wording or scope. +// +// Comment text is data. Nothing in it is executed, fetched or followed: it is +// matched against fixed phrases and, when it states a rule, quoted. +package reviewrules + +import ( + "regexp" + "strings" + "unicode/utf8" +) + +// MaxRuleRunes bounds one rule's text. A rule is a sentence or two a reviewer +// wrote, not a document. +const MaxRuleRunes = 600 + +// Kind is how a comment is classified. +type Kind string + +const ( + // KindCorrection applies to the PR the comment was left on only. + KindCorrection Kind = "correction" + // KindRule states something meant to hold for future work: a rule, a + // criterion or a source of truth. + KindRule Kind = "rule" +) + +// Names of the explicit signals Classify matches. They are part of the log +// line and the rule's provenance, so a reader can see why a comment counted. +const ( + SignalSourceOfTruth = "source-of-truth" + SignalAuthoritative = "authoritative" + SignalAlways = "always" + SignalNever = "never" + SignalGoingForward = "going-forward" + SignalCaptureRequest = "capture-request" + SignalCriteria = "assessment-criteria" + SignalKnowledgeBase = "knowledge-base" +) + +type signal struct { + name string + re *regexp.Regexp +} + +// ruleSignals are the phrases that mark a comment as a rule. Matching is +// case-insensitive and on word boundaries; adding a phrase means adding a row +// here and a row to the table test. +var ruleSignals = []signal{ + {SignalSourceOfTruth, regexp.MustCompile(`(?i)\bsources?[- ]of[- ]truth\b`)}, + {SignalAuthoritative, regexp.MustCompile(`(?i)\bauthoritative\b`)}, + {SignalAlways, regexp.MustCompile(`(?i)\b(?:should|must|needs? to|has to|have to) always\b`)}, + {SignalNever, regexp.MustCompile(`(?i)\b(?:should|must) never\b`)}, + {SignalGoingForward, regexp.MustCompile(`(?i)\b(?:going forward|from now on|in (?:the )?future|next time|for future (?:analys[ie]s|assessments|runs|prs|work))\b`)}, + {SignalCaptureRequest, regexp.MustCompile(`(?i)\b(?:capture|record|document|write down|remember)\s+(?:this|that|it)\b`)}, + {SignalCriteria, regexp.MustCompile(`(?i)\bassessment (?:criteria|rubric|knowledge|guidance)\b`)}, + {SignalKnowledgeBase, regexp.MustCompile(`(?i)\bknowledge[- ]?base\b`)}, +} + +// foreignHomeHint matches wording that puts the rule's proper home in the +// reviewed project's own repository (its docs or criteria files), which the +// hive does not write to. +var foreignHomeHint = regexp.MustCompile(`(?i)\b(?:(?:in|to|into)\s+(?:your|this|the)\s+(?:repo|repository|project)\b|your\s+(?:docs|documentation|criteria|readme|contributing))`) + +// metaClause matches a clause that asks for the capture rather than stating the +// rule ("capture this in the knowledge base"). It is dropped from the rule text +// so the stored rule is the rule. +var metaClause = regexp.MustCompile(`(?i)^\s*(?:please\s+|and\s+)?(?:capture|record|document|add|write down|remember)\s+(?:this|that|it)\b`) + +var sentenceSplit = regexp.MustCompile(`[.!?]+\s+|\n+`) + +// Classification is Classify's verdict. +type Classification struct { + Kind Kind + // Signals are the names of the signals that matched, in table order. + Signals []string +} + +// Classify says whether body states a rule for future work. It is a pure +// function of the text: every signal is an explicit phrase from ruleSignals. +func Classify(body string) Classification { + var hit []string + for _, s := range ruleSignals { + if s.re.MatchString(body) { + hit = append(hit, s.name) + } + } + if len(hit) == 0 { + return Classification{Kind: KindCorrection} + } + return Classification{Kind: KindRule, Signals: hit} +} + +// ForeignHomeHint reports whether body asks for the rule to live in the +// reviewed project's own repository. +func ForeignHomeHint(body string) bool { return foreignHomeHint.MatchString(body) } + +// ExtractRuleText returns the rule a comment states: the sentences that carry a +// signal, without the clauses that only ask for the capture, normalised and +// bounded. A comment with no signal yields its whole body, bounded. +func ExtractRuleText(body string) string { + var kept []string + for _, sent := range sentenceSplit.Split(body, -1) { + if strings.TrimSpace(sent) == "" { + continue + } + if Classify(sent).Kind == KindRule { + kept = append(kept, sent) + } + } + if len(kept) == 0 { + kept = []string{body} + } + var clauses []string + for _, sent := range kept { + for _, c := range strings.Split(sent, ";") { + if !metaClause.MatchString(c) { + clauses = append(clauses, c) + } + } + } + if len(clauses) == 0 { + clauses = kept + } + text := NormalizeText(strings.Join(clauses, "; ")) + return truncateRunes(text, MaxRuleRunes) +} + +// NormalizeText turns control characters and runs of whitespace into single +// spaces, removes HTML comment delimiters (a rule is quoted into comments and +// files, and must not be able to open or close one) and trims. +func NormalizeText(s string) string { + s = strings.ReplaceAll(s, "", "") + s = strings.Map(func(r rune) rune { + if r < 0x20 || r == 0x7f { + return ' ' + } + return r + }, s) + return strings.TrimSpace(strings.Join(strings.Fields(s), " ")) +} + +func truncateRunes(s string, n int) string { + if utf8.RuneCountInString(s) <= n { + return s + } + return string([]rune(s)[:n]) + "..." +} + +var mentionRe = regexp.MustCompile("(^|[^\\w`@])@([A-Za-z0-9][A-Za-z0-9_-]*)") + +// DefuseMentions wraps raw @handles in backticks so text quoted into a GitHub +// comment never notifies anyone. +func DefuseMentions(s string) string { + return mentionRe.ReplaceAllString(s, "$1`@$2`") +} diff --git a/src/pkg/reviewrules/classify_test.go b/src/pkg/reviewrules/classify_test.go new file mode 100644 index 0000000000..1cee913176 --- /dev/null +++ b/src/pkg/reviewrules/classify_test.go @@ -0,0 +1,112 @@ +package reviewrules + +import ( + "strings" + "testing" +) + +const maintainersComment = ".project/maintainers.yaml is the source of truth for ownership and maintainership; capture this in the assessment knowledge base" + +func TestClassify(t *testing.T) { + tests := []struct { + name string + body string + kind Kind + signals []string + }{ + {"maintainer fixture", maintainersComment, KindRule, []string{SignalSourceOfTruth, SignalCaptureRequest, SignalCriteria, SignalKnowledgeBase}}, + {"source of truth, hyphenated", "The CODEOWNERS file is the single source-of-truth here.", KindRule, []string{SignalSourceOfTruth}}, + {"authoritative", "That registry is authoritative for names.", KindRule, []string{SignalAuthoritative}}, + {"should always", "Analyses should always cite the scorecard.", KindRule, []string{SignalAlways}}, + {"must always", "You must always link the commit.", KindRule, []string{SignalAlways}}, + {"must never", "The score must never exceed the evidence.", KindRule, []string{SignalNever}}, + {"going forward", "Going forward, use the maintainers file.", KindRule, []string{SignalGoingForward}}, + {"from now on", "From now on check the license first.", KindRule, []string{SignalGoingForward}}, + {"next time", "Next time pin the link to a commit.", KindRule, []string{SignalGoingForward}}, + {"for future assessments", "Remember this for future assessments.", KindRule, []string{SignalGoingForward, SignalCaptureRequest}}, + {"document this", "Please document this somewhere.", KindRule, []string{SignalCaptureRequest}}, + {"assessment criteria", "This is missing from the assessment criteria.", KindRule, []string{SignalCriteria}}, + {"knowledge base", "Add that to the knowledgebase.", KindRule, []string{SignalKnowledgeBase}}, + {"one-off correction", "The score is too high for Agent readiness.", KindCorrection, nil}, + {"typo", "typo in line 4", KindCorrection, nil}, + {"always without a modal", "He is always late with reviews.", KindCorrection, nil}, + {"empty", "", KindCorrection, nil}, + {"upper case", "SOURCE OF TRUTH is the lockfile", KindRule, []string{SignalSourceOfTruth}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := Classify(tc.body) + if got.Kind != tc.kind { + t.Fatalf("kind = %q, want %q (signals %v)", got.Kind, tc.kind, got.Signals) + } + if strings.Join(got.Signals, ",") != strings.Join(tc.signals, ",") { + t.Fatalf("signals = %v, want %v", got.Signals, tc.signals) + } + }) + } +} + +func TestExtractRuleText(t *testing.T) { + long := strings.Repeat("always cite ", 100) + "this should always hold" + tests := []struct { + name string + body string + want string + }{ + {"fixture drops the capture request clause", maintainersComment, ".project/maintainers.yaml is the source of truth for ownership and maintainership"}, + {"keeps only signal sentences", "The score is high. The maintainers file is the source of truth. Thanks!", "The maintainers file is the source of truth"}, + {"multi-line", "First line.\nGoing forward, cite the commit.\nBye", "Going forward, cite the commit"}, + {"only a capture request keeps it", "Please capture this.", "Please capture this."}, + {"no signal returns the body", "just a remark", "just a remark"}, + {"control characters and comment delimiters", "source of truth is\tthe lockfile\x00", "source of truth x is the lockfile"}, + {"empty clause list falls back", "capture this; record it", "capture this; record it"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := ExtractRuleText(tc.body) + if strings.Join(strings.Fields(got), " ") != strings.Join(strings.Fields(tc.want), " ") { + t.Fatalf("text = %q, want %q", got, tc.want) + } + }) + } + t.Run("bounded", func(t *testing.T) { + got := ExtractRuleText(long) + if r := []rune(got); len(r) != MaxRuleRunes+len("...") || !strings.HasSuffix(got, "...") { + t.Fatalf("len = %d, want %d with ellipsis", len(r), MaxRuleRunes+3) + } + }) +} + +func TestForeignHomeHint(t *testing.T) { + tests := []struct { + body string + want bool + }{ + {"Please add this in your repo's docs", true}, + {"put it into the repository", true}, + {"This belongs in your criteria file", true}, + {"update your documentation", true}, + {"capture this in the assessment knowledge base", false}, + {"the project has maintainers", false}, + } + for _, tc := range tests { + if got := ForeignHomeHint(tc.body); got != tc.want { + t.Errorf("ForeignHomeHint(%q) = %v, want %v", tc.body, got, tc.want) + } + } +} + +func TestDefuseMentions(t *testing.T) { + tests := []struct{ in, want string }{ + {"ask @nate-double-u now", "ask `@nate-double-u` now"}, + {"@alice first", "`@alice` first"}, + {"already `@bob` quoted", "already `@bob` quoted"}, + {"mail a@b.example", "mail a@b.example"}, + {"no mention", "no mention"}, + } + for _, tc := range tests { + if got := DefuseMentions(tc.in); got != tc.want { + t.Errorf("DefuseMentions(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} diff --git a/src/pkg/reviewrules/notices.go b/src/pkg/reviewrules/notices.go new file mode 100644 index 0000000000..f6489044f1 --- /dev/null +++ b/src/pkg/reviewrules/notices.go @@ -0,0 +1,56 @@ +package reviewrules + +import ( + "fmt" + "strings" +) + +const ( + // AckMarkerPrefix opens the hidden marker of the acknowledgement comment. + AckMarkerPrefix = "\n", AckMarkerPrefix, strings.Join(ids, ",")) + for _, a := range acks { + r := a.Rule + if a.Merged { + fmt.Fprintf(&b, "This restates rule %s, which the hive already recorded in its knowledge and applies to %s. Your comment (%s) was added to that rule's sources.\n\n", r.ID, r.Describe(), a.URL) + } else { + fmt.Fprintf(&b, "Recorded as rule %s in the hive's knowledge, applies to %s. Source: %s\n\n", r.ID, r.Describe(), a.URL) + } + b.WriteString(fenced(r.Text)) + b.WriteString("\n") + } + b.WriteString("Recorded rules are loaded into the matching agent's next runs, so they hold for future work and not only for this PR. If a rule is not what you meant, reply here and say so; an operator can retract it with `hivectl knowledge state deprecated` (the entry id is `rule-` followed by the lowercase rule id, for example `rule-r-1`).\n") + return b.String() +} + +// HomeProposalNote is the one needs-human note for a rule whose proper home is +// a repo the hive does not manage. It carries the exact proposed text and the +// source comment; the hive never writes to that repo. +func HomeProposalNote(r *Rule) string { + var b strings.Builder + fmt.Fprintf(&b, "%s rule=%s -->\n", HomeMarkerPrefix, r.ID) + fmt.Fprintf(&b, "Needs a human: reviewer feedback here states a rule whose proper home is `%s`'s own files (for example its assessment criteria), and the hive does not write to that repo. The hive has recorded it in its own knowledge as rule %s so its own work follows it. To make it part of the project, a maintainer can add this text where the project keeps such criteria:\n\n", r.ForeignHome, r.ID) + b.WriteString(fenced(r.Text)) + if len(r.Provenance) > 0 { + fmt.Fprintf(&b, "\nSource: %s\n", r.Provenance[0].CommentURL) + } + return b.String() +} diff --git a/src/pkg/reviewrules/notices_test.go b/src/pkg/reviewrules/notices_test.go new file mode 100644 index 0000000000..9c89d33c21 --- /dev/null +++ b/src/pkg/reviewrules/notices_test.go @@ -0,0 +1,54 @@ +package reviewrules + +import ( + "strings" + "testing" + "time" +) + +func TestAckComment(t *testing.T) { + r := &Rule{ID: "R-1", Text: "Ask @nate-double-u; see ``` fence", Scope: ScopeAgent, AppliesTo: "quality"} + got := AckComment([]Ack{{Rule: r, URL: "https://github.com/cncf/techdocs/pull/424#discussion_r1"}}) + for _, want := range []string{AckMarkerPrefix + " ids=R-1 -->", "Recorded as rule R-1 in the hive's knowledge, applies to the `quality` agent's future work.", + "Source: https://github.com/cncf/techdocs/pull/424#discussion_r1", "`@nate-double-u`", "'''", "hivectl knowledge state", "rule-r-1"} { + if !strings.Contains(got, want) { + t.Errorf("ack missing %q:\n%s", want, got) + } + } + if strings.Contains(got, "```text\nAsk @") || strings.Count(got, "```") != 2 { + t.Errorf("rule text must sit in one fence it cannot close:\n%s", got) + } + if strings.Contains(got, "\u2014") { + t.Error("no em-dashes") + } + merged := AckComment([]Ack{{Rule: r, URL: "u1", Merged: true}, {Rule: &Rule{ID: "R-2", Text: "t", Scope: ScopeGlobal}, URL: "u2"}}) + if !strings.Contains(merged, "This restates rule R-1") || !strings.Contains(merged, "ids=R-1,R-2") || !strings.Contains(merged, "Recorded as rule R-2") { + t.Errorf("merged ack:\n%s", merged) + } +} + +func TestHomeProposalNote(t *testing.T) { + r := &Rule{ID: "R-3", Text: ".project/maintainers.yaml is the source of truth for ownership", ForeignHome: "cncf/techdocs", + Provenance: []Provenance{{CommentURL: "https://github.com/cncf/techdocs/pull/424#discussion_r1"}}} + got := HomeProposalNote(r) + for _, want := range []string{HomeMarkerPrefix + " rule=R-3 -->", "Needs a human", "`cncf/techdocs`", "does not write to that repo", + "```text\n.project/maintainers.yaml is the source of truth for ownership\n```", "Source: https://github.com/cncf/techdocs/pull/424#discussion_r1"} { + if !strings.Contains(got, want) { + t.Errorf("note missing %q:\n%s", want, got) + } + } + r.Provenance = nil + if strings.Contains(HomeProposalNote(r), "Source:") { + t.Error("no provenance, no source line") + } +} + +func TestNoticesAreOnePerRuleThroughTheStore(t *testing.T) { + s, _, _ := openStore(t, Config{}) + c := cmt("inline:1", "OWNER", "Going forward, put the criteria in your docs: the maintainers file is the source of truth.") + s.CaptureComment(c, t0) + s.CaptureComment(c, t0.Add(time.Minute)) + if n := len(s.HomeProposals()); n != 1 { + t.Fatalf("proposals = %d, want exactly one for the one rule", n) + } +} diff --git a/src/pkg/reviewrules/store.go b/src/pkg/reviewrules/store.go new file mode 100644 index 0000000000..2ac07fc03e --- /dev/null +++ b/src/pkg/reviewrules/store.go @@ -0,0 +1,853 @@ +package reviewrules + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +const ( + // DirEnvVar names the directory the rule index lives in. It is the review + // feedback sweep's own state directory, so one volume holds both. + DirEnvVar = "HIVE_REVIEW_FEEDBACK_DIR" + // DefaultDir sits on the /data volume so the index survives pod rolls. + DefaultDir = "/data/review-feedback" + // VaultEnvVar overrides the knowledge vault the rules are written to. + VaultEnvVar = "HIVE_REVIEW_FEEDBACK_RULES_VAULT" + // DefaultVaultDir is the local knowledge channel the rules live in. + DefaultVaultDir = "/data/vaults/review-rules" + // VaultName is the channel name the vault is connected under. + VaultName = "review-rules" + // DisableEnvVar turns rule capture off when set to a false-like value. The + // sweep itself keeps running. + DisableEnvVar = "HIVE_REVIEW_FEEDBACK_RULES" + // TrustedAssociationsEnvVar lists the author associations whose rules are + // captured at once; every other reviewer's rule is queued for a person. + TrustedAssociationsEnvVar = "HIVE_REVIEW_FEEDBACK_RULE_TRUSTED_ASSOCIATIONS" + // DefaultTrustedAssociations are the associations that may state a rule + // on their own authority. CONTRIBUTOR is left out: the sweep routes a + // contributor's feedback, but a rule changes what every later run is told. + DefaultTrustedAssociations = "OWNER,MEMBER,COLLABORATOR" + // MaxRulesEnvVar bounds how many live (active or pending) rules are kept. + MaxRulesEnvVar = "HIVE_REVIEW_FEEDBACK_RULES_MAX" + // DefaultMaxRules keeps the pinned set small enough to read. + DefaultMaxRules = 200 + // MergeThresholdEnvVar overrides how similar two rules' texts must be to + // merge, as a fraction between 0 and 1. + MergeThresholdEnvVar = "HIVE_REVIEW_FEEDBACK_RULE_MERGE_THRESHOLD" + // DefaultMergeThreshold is the token-overlap (Jaccard) at which two rules + // of the same scope are the same rule. + DefaultMergeThreshold = 0.8 + + indexFile = "rules.json" + + // Confidence levels recorded with a rule: what the hive can claim about + // how sure it is that the text is what the reviewer meant. + confidenceSweep = 0.85 + confidenceAgent = 0.9 + confidencePending = 0.6 + confidenceAgentFromSeen = 0.7 + + // maxSeenComments bounds the remembered review comments. + maxSeenComments = 500 + // maxRepliesKept bounds the remembered applied agent replies. + maxRepliesKept = 500 + // titleRunes bounds a rule's entry title. + titleRunes = 70 + // vaultAuthor is the author recorded on rule entries. No agent has this + // name, so a pending (draft) rule never reaches an agent through the + // author-matched draft path. + vaultAuthor = "review-feedback" +) + +// Status is where a rule is in its life. +type Status string + +const ( + // StatusActive rules are approved entries the primer loads. + StatusActive Status = "active" + // StatusPending rules wait for a person: they are draft entries. + StatusPending Status = "pending" + // StatusRetracted rules were withdrawn, by a person or by removing or + // deprecating the entry. + StatusRetracted Status = "retracted" + // StatusMerged rules were found to restate another rule. + StatusMerged Status = "merged" +) + +// Outcome is what a capture did. +type Outcome string + +const ( + OutcomeNotRule Outcome = "not_rule" + OutcomeCaptured Outcome = "captured" + OutcomeQueued Outcome = "queued" + OutcomeMerged Outcome = "merged" + OutcomeDuplicate Outcome = "duplicate" + OutcomeRefused Outcome = "refused" + OutcomeRefined Outcome = "refined" + OutcomeRejected Outcome = "rejected" +) + +// ErrFull is returned when the live rule set is at its bound. +var ErrFull = errors.New("the review-rule set is full; retract a rule to make room") + +// Provenance records one comment a rule came from. +type Provenance struct { + Repo string `json:"repo"` + PR int `json:"pr"` + PRURL string `json:"pr_url"` + CommentID string `json:"comment_id"` + CommentURL string `json:"comment_url"` + Reviewer string `json:"reviewer"` + Association string `json:"association"` + At time.Time `json:"at"` + // Source is "sweep" for a rule the sweep extracted from the comment and + // "agent" for one the kicked agent stated in its reply block. + Source string `json:"source"` + // Merged marks a comment that restated a rule that already existed. + Merged bool `json:"merged,omitempty"` + // Acked is set once the reviewer was told on the PR, or when no + // notice is owed (a queued rule). + Acked bool `json:"acked,omitempty"` +} + +// Rule is one captured rule. +type Rule struct { + ID string `json:"id"` + Num int `json:"num"` + Text string `json:"text"` + Scope Scope `json:"scope"` + AppliesTo string `json:"applies_to,omitempty"` + Lane string `json:"lane,omitempty"` + Status Status `json:"status"` + RetractedBy string `json:"retracted_by,omitempty"` + MergedInto string `json:"merged_into,omitempty"` + Confidence float64 `json:"confidence"` + Provenance []Provenance `json:"provenance"` + ForeignHome string `json:"foreign_home,omitempty"` + HomeNoted bool `json:"home_noted,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// Slug is the knowledge entry id of the rule. +func (r *Rule) Slug() string { return "rule-" + strings.ToLower(r.ID) } + +type seenComment struct { + CommentID string `json:"comment_id"` + URL string `json:"url"` + Reviewer string `json:"reviewer"` + Association string `json:"association"` + Repo string `json:"repo"` + PR int `json:"pr"` + Lane string `json:"lane"` + At time.Time `json:"at"` +} + +type index struct { + Next int `json:"next"` + Rules []*Rule `json:"rules"` + Comments map[string]string `json:"comments"` + Seen map[string]seenComment `json:"seen"` + Replies map[string]time.Time `json:"replies"` +} + +// Config is the capture policy. +type Config struct { + // Trusted are the upper-case author associations that may state a rule + // on their own authority. + Trusted map[string]bool + // MaxRules bounds live rules. + MaxRules int + // MergeThreshold is the text similarity at which same-scope rules merge. + MergeThreshold float64 +} + +// ConfigFromEnv reads the knobs, falling back to the Default* constants. +func ConfigFromEnv() Config { + cfg := Config{MaxRules: DefaultMaxRules, MergeThreshold: DefaultMergeThreshold} + list := strings.TrimSpace(os.Getenv(TrustedAssociationsEnvVar)) + if list == "" { + list = DefaultTrustedAssociations + } + cfg.Trusted = ParseAssociations(list) + if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv(MaxRulesEnvVar))); err == nil && n > 0 { + cfg.MaxRules = n + } + if f, err := strconv.ParseFloat(strings.TrimSpace(os.Getenv(MergeThresholdEnvVar)), 64); err == nil && f > 0 && f <= 1 { + cfg.MergeThreshold = f + } + return cfg +} + +// ParseAssociations parses a comma-separated association list. +func ParseAssociations(list string) map[string]bool { + out := map[string]bool{} + for _, a := range strings.Split(list, ",") { + if a = strings.ToUpper(strings.TrimSpace(a)); a != "" { + out[a] = true + } + } + return out +} + +// Normalized fills unset knobs with the defaults. +func (c Config) Normalized() Config { + if c.Trusted == nil { + c.Trusted = ParseAssociations(DefaultTrustedAssociations) + } + if c.MaxRules <= 0 { + c.MaxRules = DefaultMaxRules + } + if c.MergeThreshold <= 0 || c.MergeThreshold > 1 { + c.MergeThreshold = DefaultMergeThreshold + } + return c +} + +// Enabled reports whether rule capture runs. It is on by default; set +// HIVE_REVIEW_FEEDBACK_RULES to 0, false, off or no to stop it. +func Enabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv(DisableEnvVar))) { + case "0", "false", "off", "no": + return false + } + return true +} + +// Dir resolves the index directory. +func Dir() string { + if d := strings.TrimSpace(os.Getenv(DirEnvVar)); d != "" { + return d + } + return DefaultDir +} + +// VaultDir resolves the knowledge vault the rules are written to. +func VaultDir() string { + if d := strings.TrimSpace(os.Getenv(VaultEnvVar)); d != "" { + return d + } + return DefaultVaultDir +} + +// Store persists captured rules. +type Store struct { + mu sync.Mutex + dir string + vault string + cfg Config + st index + // Reindex, when set, is called after the vault changed so the primer's + // view is current without waiting for its periodic refresh. + Reindex func() +} + +// Open loads the index from dir and writes entries to vault. A missing index +// is an empty store; an unreadable one is returned with the error and still +// usable in memory. +func Open(dir, vault string, cfg Config) (*Store, error) { + s := &Store{dir: dir, vault: vault, cfg: cfg.Normalized()} + s.st = newIndex() + b, err := os.ReadFile(filepath.Join(dir, indexFile)) + if err != nil { + if os.IsNotExist(err) { + return s, nil + } + return s, err + } + var loaded index + if err := json.Unmarshal(b, &loaded); err != nil { + return s, err + } + s.st = loaded + s.fillMaps() + return s, nil +} + +func newIndex() index { + return index{Next: 1, Comments: map[string]string{}, Seen: map[string]seenComment{}, Replies: map[string]time.Time{}} +} + +func (s *Store) fillMaps() { + if s.st.Next < 1 { + s.st.Next = 1 + } + if s.st.Comments == nil { + s.st.Comments = map[string]string{} + } + if s.st.Seen == nil { + s.st.Seen = map[string]seenComment{} + } + if s.st.Replies == nil { + s.st.Replies = map[string]time.Time{} + } +} + +// Comment is one review comment offered for capture. +type Comment struct { + Repo string + PR int + CommentID string + URL string + Reviewer string + Association string + Body string + At time.Time + // Lane is the base name of the agent that authored the PR; a rule with no + // sharper scope applies to that agent's future work. + Lane string + // ManagedRepo says whether the hive manages Repo. A rule whose proper + // home is the reviewed project's own files is only proposed, never + // written, when the hive does not manage it. + ManagedRepo bool +} + +func (c Comment) key() string { return commentKey(c.Repo, c.PR, c.CommentID) } + +func commentKey(repo string, pr int, id string) string { + return fmt.Sprintf("%s#%d:%s", repo, pr, id) +} + +func (c Comment) prURL() string { return fmt.Sprintf("https://github.com/%s/pull/%d", c.Repo, c.PR) } + +// Result is what one capture did. +type Result struct { + Outcome Outcome + // Rule is a copy of the rule the comment became or joined; nil for + // OutcomeNotRule and OutcomeRefused. + Rule *Rule + // Signals are the classification signals that matched. + Signals []string + Err error +} + +func (s *Store) trusted(assoc string) bool { + return s.cfg.Trusted[strings.ToUpper(strings.TrimSpace(assoc))] +} + +func (s *Store) live() int { + n := 0 + for _, r := range s.st.Rules { + if r.Status == StatusActive || r.Status == StatusPending { + n++ + } + } + return n +} + +func (s *Store) byID(id string) *Rule { + for _, r := range s.st.Rules { + if r.ID == id { + return r + } + } + return nil +} + +// resolve follows MergedInto to the rule a comment finally belongs to. +func (s *Store) resolve(r *Rule) *Rule { + for i := 0; r != nil && r.Status == StatusMerged && i < len(s.st.Rules); i++ { + r = s.byID(r.MergedInto) + } + return r +} + +func scopeKey(sc Scope, appliesTo string) string { + return string(sc) + "|" + strings.ToLower(appliesTo) +} + +func (r *Rule) scopeKey() string { return scopeKey(r.Scope, r.AppliesTo) } + +var tokenRe = regexp.MustCompile(`[a-z0-9_./-]+`) + +func tokens(s string) map[string]bool { + out := map[string]bool{} + for _, t := range tokenRe.FindAllString(strings.ToLower(s), -1) { + if t = strings.Trim(t, "./-"); t != "" { + out[t] = true + } + } + return out +} + +// similarity is the Jaccard overlap of two texts' token sets. +func similarity(a, b string) float64 { + ta, tb := tokens(a), tokens(b) + if len(ta) == 0 || len(tb) == 0 { + return 0 + } + inter := 0 + for t := range ta { + if tb[t] { + inter++ + } + } + return float64(inter) / float64(len(ta)+len(tb)-inter) +} + +// nearDuplicate finds a live rule of the same scope whose text restates text. +func (s *Store) nearDuplicate(text, key string, skip *Rule) *Rule { + for _, r := range s.st.Rules { + if r == skip || (r.Status != StatusActive && r.Status != StatusPending) || r.scopeKey() != key { + continue + } + if similarity(r.Text, text) >= s.cfg.MergeThreshold { + return r + } + } + return nil +} + +// Remember records that the hive saw a review comment, so an agent's reply that +// cites it as evidence can be tied to its reviewer and association. +func (s *Store) Remember(c Comment, now time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + if c.URL == "" { + return nil + } + return s.txn(func() []*Rule { + s.st.Seen[c.URL] = seenComment{CommentID: c.CommentID, URL: c.URL, Reviewer: c.Reviewer, Association: c.Association, + Repo: c.Repo, PR: c.PR, Lane: c.Lane, At: now} + if len(s.st.Seen) > maxSeenComments { + urls := make([]string, 0, len(s.st.Seen)) + for u := range s.st.Seen { + urls = append(urls, u) + } + sort.Slice(urls, func(i, j int) bool { return s.st.Seen[urls[i]].At.Before(s.st.Seen[urls[j]].At) }) + for _, u := range urls[:len(s.st.Seen)-maxSeenComments] { + delete(s.st.Seen, u) + } + } + return nil + }) +} + +// CaptureComment classifies a comment and, when it states a rule, captures it. +// It is idempotent per comment: a comment id that was captured, queued or +// merged before returns OutcomeDuplicate and changes nothing. +func (s *Store) CaptureComment(c Comment, now time.Time) Result { + cl := Classify(c.Body) + if cl.Kind != KindRule { + return Result{Outcome: OutcomeNotRule} + } + s.mu.Lock() + defer s.mu.Unlock() + if id, ok := s.st.Comments[c.key()]; ok { + return Result{Outcome: OutcomeDuplicate, Rule: s.copyOf(s.resolve(s.byID(id))), Signals: cl.Signals} + } + text := ExtractRuleText(c.Body) + if text == "" { + return Result{Outcome: OutcomeRefused, Signals: cl.Signals, Err: errors.New("the comment has no text to record")} + } + home := "" + if !c.ManagedRepo && ForeignHomeHint(c.Body) { + home = c.Repo + } + return s.captureLocked(c, text, ScopeAgent, c.Lane, home, "sweep", confidenceSweep, cl.Signals, now) +} + +// captureLocked creates a rule, or joins an existing near-identical one. +func (s *Store) captureLocked(c Comment, text string, sc Scope, appliesTo, home, source string, conf float64, signals []string, now time.Time) Result { + trusted := s.trusted(c.Association) + prov := Provenance{Repo: c.Repo, PR: c.PR, PRURL: c.prURL(), CommentID: c.CommentID, CommentURL: c.URL, + Reviewer: c.Reviewer, Association: strings.ToUpper(c.Association), At: now, Source: source} + key := scopeKey(sc, appliesTo) + var res Result + err := s.txn(func() []*Rule { + if dup := s.nearDuplicate(text, key, nil); dup != nil { + prov.Merged = true + prov.Acked = !trusted + dup.Provenance = append(dup.Provenance, prov) + if trusted && dup.Status == StatusPending { + dup.Status = StatusActive + } + if home != "" && dup.ForeignHome == "" { + dup.ForeignHome = home + } + dup.UpdatedAt = now + s.st.Comments[c.key()] = dup.ID + res = Result{Outcome: OutcomeMerged, Rule: s.copyOf(dup), Signals: signals} + return []*Rule{dup} + } + if s.live() >= s.cfg.MaxRules { + res = Result{Outcome: OutcomeRefused, Signals: signals, Err: ErrFull} + return nil + } + r := &Rule{ID: "R-" + strconv.Itoa(s.st.Next), Num: s.st.Next, Text: text, Scope: sc, AppliesTo: appliesTo, Lane: c.Lane, + Status: StatusActive, Confidence: conf, ForeignHome: home, CreatedAt: now, UpdatedAt: now} + s.st.Next++ + outcome := OutcomeCaptured + if !trusted { + r.Status, r.Confidence, outcome = StatusPending, confidencePending, OutcomeQueued + prov.Acked = true + } + r.Provenance = []Provenance{prov} + s.st.Rules = append(s.st.Rules, r) + s.st.Comments[c.key()] = r.ID + res = Result{Outcome: outcome, Rule: s.copyOf(r), Signals: signals} + return []*Rule{r} + }) + if err != nil { + return Result{Outcome: OutcomeRefused, Signals: signals, Err: err} + } + return res +} + +func (s *Store) copyOf(r *Rule) *Rule { + if r == nil { + return nil + } + cp := *r + cp.Provenance = append([]Provenance(nil), r.Provenance...) + return &cp +} + +// Applied is what one entry of an agent's block did. +type Applied struct { + Outcome Outcome + Rule *Rule + Reason string +} + +// ApplyReply applies the structured block in an agent reply. replyID names the +// reply, so the same reply is applied once. An entry must cite, as its +// evidence, a review comment the hive saw (Remember): the reviewer's trust +// comes from that comment, never from anything the block claims. managed says +// whether the hive manages a repo. +func (s *Store) ApplyReply(replyID, body string, managed func(repo string) bool, now time.Time) []Applied { + s.mu.Lock() + defer s.mu.Unlock() + if _, done := s.st.Replies[replyID]; done { + return nil + } + rules, errs := ParseBlocks(body) + var out []Applied + for _, e := range errs { + out = append(out, Applied{Outcome: OutcomeRejected, Reason: e.Error()}) + } + for _, br := range rules { + out = append(out, s.applyOneLocked(br, managed, now)) + } + if len(rules)+len(errs) > 0 || strings.Contains(body, BlockMarker) { + _ = s.txn(func() []*Rule { + s.st.Replies[replyID] = now + if len(s.st.Replies) > maxRepliesKept { + ids := make([]string, 0, len(s.st.Replies)) + for id := range s.st.Replies { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { return s.st.Replies[ids[i]].Before(s.st.Replies[ids[j]]) }) + for _, id := range ids[:len(s.st.Replies)-maxRepliesKept] { + delete(s.st.Replies, id) + } + } + return nil + }) + } + return out +} + +func (s *Store) applyOneLocked(br BlockRule, managed func(string) bool, now time.Time) Applied { + seen, ok := s.st.Seen[br.EvidenceURL] + if !ok { + return Applied{Outcome: OutcomeRejected, Reason: "evidence_url does not name a review comment the hive saw"} + } + sc, appliesTo := Scope(br.Scope), br.AppliesTo + if sc == "" { + sc = ScopeAgent + } + switch sc { + case ScopeAgent: + if appliesTo == "" { + appliesTo = seen.Lane + } + case ScopeGlobal: + appliesTo = "" + } + home := "" + if sc == ScopeRepo && managed != nil && !managed(appliesTo) { + home = appliesTo + } + key := commentKey(seen.Repo, seen.PR, seen.CommentID) + c := Comment{Repo: seen.Repo, PR: seen.PR, CommentID: seen.CommentID, URL: seen.URL, Reviewer: seen.Reviewer, + Association: seen.Association, Lane: seen.Lane} + id, known := s.st.Comments[key] + if !known { + res := s.captureLocked(c, br.Text, sc, appliesTo, home, "agent", confidenceAgentFromSeen, nil, now) + return Applied{Outcome: res.Outcome, Rule: res.Rule, Reason: errText(res.Err)} + } + r := s.resolve(s.byID(id)) + if r == nil || (r.Status != StatusActive && r.Status != StatusPending) { + return Applied{Outcome: OutcomeRejected, Reason: "the rule this comment became is no longer live"} + } + var out Applied + err := s.txn(func() []*Rule { + touched := []*Rule{r} + r.Text, r.Scope, r.AppliesTo, r.UpdatedAt = br.Text, sc, appliesTo, now + if s.trusted(seen.Association) { + r.Confidence = confidenceAgent + } + if home != "" && r.ForeignHome != home { + r.ForeignHome, r.HomeNoted = home, false + } + out = Applied{Outcome: OutcomeRefined, Rule: s.copyOf(r)} + if dup := s.nearDuplicate(r.Text, r.scopeKey(), r); dup != nil { + keep, drop := dup, r + if drop.Num < keep.Num { + keep, drop = drop, keep + } + drop.Status, drop.MergedInto, drop.UpdatedAt = StatusMerged, keep.ID, now + keep.Provenance = append(keep.Provenance, drop.Provenance...) + if keep.Status == StatusPending && s.anyTrusted(keep.Provenance) { + keep.Status = StatusActive + } + if keep.ForeignHome == "" { + keep.ForeignHome = drop.ForeignHome + } + for ck, rid := range s.st.Comments { + if rid == drop.ID { + s.st.Comments[ck] = keep.ID + } + } + touched = []*Rule{keep, drop} + out = Applied{Outcome: OutcomeMerged, Rule: s.copyOf(keep)} + } + return touched + }) + if err != nil { + return Applied{Outcome: OutcomeRejected, Reason: err.Error()} + } + return out +} + +func (s *Store) anyTrusted(p []Provenance) bool { + for _, pr := range p { + if s.trusted(pr.Association) { + return true + } + } + return false +} + +func errText(err error) string { + if err == nil { + return "" + } + return err.Error() +} + +// Retract withdraws a rule: its entry stays in the vault as deprecated, so the +// history is readable, and the primer stops loading it. +func (s *Store) Retract(id, by string, now time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + r := s.byID(strings.ToUpper(strings.TrimSpace(id))) + if r == nil { + return fmt.Errorf("no rule %q", id) + } + if r.Status == StatusRetracted { + return nil + } + return s.txn(func() []*Rule { + r.Status, r.RetractedBy, r.UpdatedAt = StatusRetracted, by, now + return []*Rule{r} + }) +} + +// Get returns a copy of the rule with this id, or nil. +func (s *Store) Get(id string) *Rule { + s.mu.Lock() + defer s.mu.Unlock() + return s.copyOf(s.byID(strings.ToUpper(strings.TrimSpace(id)))) +} + +// List returns copies of every rule, in id order. +func (s *Store) List() []*Rule { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]*Rule, 0, len(s.st.Rules)) + for _, r := range s.st.Rules { + out = append(out, s.copyOf(r)) + } + return out +} + +// Ack is a capture the reviewer has not been told about yet. +type Ack struct { + Rule *Rule + CommentID string + URL string + // Merged is true when the comment restated an existing rule. + Merged bool +} + +// UnackedFor lists the captures from PR repo#pr whose reviewer was not told. +func (s *Store) UnackedFor(repo string, pr int) []Ack { + s.mu.Lock() + defer s.mu.Unlock() + var out []Ack + for _, r := range s.st.Rules { + if r.Status != StatusActive { + continue + } + for _, p := range r.Provenance { + if p.Repo == repo && p.PR == pr && !p.Acked { + out = append(out, Ack{Rule: s.copyOf(r), CommentID: p.CommentID, URL: p.CommentURL, Merged: p.Merged}) + } + } + } + return out +} + +// MarkAcked records that the reviewer was told about these comments' captures. +func (s *Store) MarkAcked(repo string, pr int, commentIDs []string) error { + s.mu.Lock() + defer s.mu.Unlock() + want := map[string]bool{} + for _, id := range commentIDs { + want[id] = true + } + return s.txn(func() []*Rule { + for _, r := range s.st.Rules { + for i := range r.Provenance { + p := &r.Provenance[i] + if p.Repo == repo && p.PR == pr && want[p.CommentID] { + p.Acked = true + } + } + } + return nil + }) +} + +// HomeProposals lists live rules whose proper home is a repo the hive does not +// manage and whose proposal was not posted yet. One proposal per rule. +func (s *Store) HomeProposals() []*Rule { + s.mu.Lock() + defer s.mu.Unlock() + var out []*Rule + for _, r := range s.st.Rules { + if r.Status == StatusActive && r.ForeignHome != "" && !r.HomeNoted { + out = append(out, s.copyOf(r)) + } + } + return out +} + +// MarkHomeNoted records the proposal for rule id was posted. +func (s *Store) MarkHomeNoted(id string) error { + s.mu.Lock() + defer s.mu.Unlock() + r := s.byID(id) + if r == nil { + return fmt.Errorf("no rule %q", id) + } + return s.txn(func() []*Rule { + r.HomeNoted = true + return nil + }) +} + +// Reconcile reads the lifecycle state of each rule's vault entry and brings the +// index in line, so an operator's `hivectl knowledge state deprecated`, or +// deleting the entry, retracts the rule, and approving a queued (draft) entry +// activates it. It returns one line per change, for the log. +func (s *Store) Reconcile(now time.Time) []string { + s.mu.Lock() + defer s.mu.Unlock() + var changes []string + var touched []*Rule + for _, r := range s.st.Rules { + if r.Status != StatusActive && r.Status != StatusPending { + continue + } + state, found := s.entryState(r) + switch { + case !found: + r.Status, r.RetractedBy = StatusRetracted, "knowledge entry removed" + changes = append(changes, r.ID+" retracted: its knowledge entry was removed") + case state == "deprecated" || state == "superseded": + r.Status, r.RetractedBy = StatusRetracted, "knowledge entry "+state + changes = append(changes, r.ID+" retracted: its knowledge entry is "+state) + case state == "draft" && r.Status == StatusActive: + r.Status = StatusPending + changes = append(changes, r.ID+" moved back to the review queue: its knowledge entry is a draft") + case state == "approved" && r.Status == StatusPending: + r.Status = StatusActive + for i := range r.Provenance { + r.Provenance[i].Acked = false + } + changes = append(changes, r.ID+" confirmed by a person: its knowledge entry is approved") + default: + continue + } + r.UpdatedAt = now + touched = append(touched, r) + } + if len(touched) > 0 { + if err := s.saveIndex(); err != nil { + changes = append(changes, "index not saved: "+err.Error()) + } + } + return changes +} + +// txn runs a mutation, then writes the touched rules' entries and the index. +// When a write fails the in-memory state is restored, so memory never claims +// something the disk does not hold. The caller holds s.mu. +func (s *Store) txn(fn func() []*Rule) error { + snap, err := json.Marshal(s.st) + if err != nil { + return err + } + touched := fn() + werr := s.flush(touched) + if werr != nil { + var restored index + if json.Unmarshal(snap, &restored) == nil { + s.st = restored + s.fillMaps() + } + } + return werr +} + +func (s *Store) flush(touched []*Rule) error { + if len(touched) > 0 { + for _, r := range touched { + if err := s.writeEntry(r); err != nil { + return err + } + } + if s.Reindex != nil { + s.Reindex() + } + } + return s.saveIndex() +} + +func (s *Store) saveIndex() error { + if err := os.MkdirAll(s.dir, 0o755); err != nil { + return err + } + b, err := json.MarshalIndent(s.st, "", " ") + if err != nil { + return err + } + tmp := filepath.Join(s.dir, indexFile+".tmp") + if err := os.WriteFile(tmp, b, 0o644); err != nil { + return err + } + return os.Rename(tmp, filepath.Join(s.dir, indexFile)) +} diff --git a/src/pkg/reviewrules/store_test.go b/src/pkg/reviewrules/store_test.go new file mode 100644 index 0000000000..0d59f7b94d --- /dev/null +++ b/src/pkg/reviewrules/store_test.go @@ -0,0 +1,496 @@ +package reviewrules + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +var t0 = time.Date(2026, 11, 18, 18, 0, 0, 0, time.UTC) + +func openStore(t *testing.T, cfg Config) (*Store, string, string) { + t.Helper() + dir, vault := filepath.Join(t.TempDir(), "idx"), filepath.Join(t.TempDir(), "vault") + s, err := Open(dir, vault, cfg) + if err != nil { + t.Fatal(err) + } + return s, dir, vault +} + +func cmt(id, assoc, body string) Comment { + return Comment{Repo: "cncf/techdocs", PR: 424, CommentID: id, URL: "https://github.com/cncf/techdocs/pull/424#discussion_r" + strings.TrimPrefix(id, "inline:"), + Reviewer: "nate-double-u", Association: assoc, Body: body, At: t0, Lane: "quality"} +} + +func readFile(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func TestCaptureTheMaintainersFileRule(t *testing.T) { + s, _, vault := openStore(t, Config{}) + res := s.CaptureComment(cmt("inline:1001", "OWNER", maintainersComment), t0) + if res.Outcome != OutcomeCaptured || res.Rule == nil || res.Err != nil { + t.Fatalf("result = %+v, want a captured rule", res) + } + r := res.Rule + if r.ID != "R-1" || r.Status != StatusActive || r.Scope != ScopeAgent || r.AppliesTo != "quality" || r.Lane != "quality" { + t.Fatalf("rule = %+v", r) + } + if r.Text != ".project/maintainers.yaml is the source of truth for ownership and maintainership" { + t.Fatalf("text = %q", r.Text) + } + if len(r.Provenance) != 1 { + t.Fatalf("provenance = %+v", r.Provenance) + } + p := r.Provenance[0] + if p.PRURL != "https://github.com/cncf/techdocs/pull/424" || p.CommentID != "inline:1001" || p.Reviewer != "nate-double-u" || + p.Association != "OWNER" || !p.At.Equal(t0) || p.Source != "sweep" || p.CommentURL == "" { + t.Fatalf("provenance = %+v", p) + } + entry := readFile(t, filepath.Join(vault, "rule-r-1.md")) + for _, want := range []string{"state: approved", "type: reviewer-rule", "tags: [review-rule, r-1, pin, reviewer:nate-double-u]", + "source: https://github.com/cncf/techdocs/pull/424#discussion_r1001", "scope:\n lanes: [quality]", "author: review-feedback", + "confidence: 0.85", "reviewer `nate-double-u` (owner)", "hivectl knowledge state rule-r-1 deprecated"} { + if !strings.Contains(entry, want) { + t.Errorf("entry missing %q:\n%s", want, entry) + } + } +} + +func TestCaptureTrustGate(t *testing.T) { + tests := []struct { + name string + cfg Config + assoc string + status Status + out Outcome + state string + }{ + {"owner", Config{}, "OWNER", StatusActive, OutcomeCaptured, "approved"}, + {"member lower case", Config{}, "member", StatusActive, OutcomeCaptured, "approved"}, + {"collaborator", Config{}, "COLLABORATOR", StatusActive, OutcomeCaptured, "approved"}, + {"contributor is queued", Config{}, "CONTRIBUTOR", StatusPending, OutcomeQueued, "draft"}, + {"none is queued", Config{}, "NONE", StatusPending, OutcomeQueued, "draft"}, + {"unknown is queued", Config{}, "", StatusPending, OutcomeQueued, "draft"}, + {"configured contributor trusted", Config{Trusted: ParseAssociations(" contributor , owner ")}, "CONTRIBUTOR", StatusActive, OutcomeCaptured, "approved"}, + {"configured list excludes member", Config{Trusted: ParseAssociations("OWNER")}, "MEMBER", StatusPending, OutcomeQueued, "draft"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s, _, vault := openStore(t, tc.cfg) + res := s.CaptureComment(cmt("inline:1", tc.assoc, maintainersComment), t0) + if res.Outcome != tc.out || res.Rule.Status != tc.status { + t.Fatalf("outcome=%s status=%s, want %s/%s", res.Outcome, res.Rule.Status, tc.out, tc.status) + } + if !strings.Contains(readFile(t, filepath.Join(vault, "rule-r-1.md")), "state: "+tc.state+"\n") { + t.Fatalf("entry state is not %s", tc.state) + } + if tc.status == StatusPending { + if got := s.UnackedFor("cncf/techdocs", 424); len(got) != 0 { + t.Fatalf("a queued rule owes the reviewer no acknowledgement: %+v", got) + } + if res.Rule.Confidence >= confidenceSweep { + t.Fatalf("queued confidence = %v", res.Rule.Confidence) + } + } + }) + } +} + +func TestCaptureOnlyRules(t *testing.T) { + s, _, vault := openStore(t, Config{}) + res := s.CaptureComment(cmt("inline:2", "OWNER", "The score is too high."), t0) + if res.Outcome != OutcomeNotRule || res.Rule != nil || len(s.List()) != 0 { + t.Fatalf("a correction must not become a rule: %+v", res) + } + if _, err := os.Stat(vault); err == nil { + t.Fatal("a correction must not touch the vault") + } +} + +func TestCaptureIsIdempotentAcrossReload(t *testing.T) { + s, dir, vault := openStore(t, Config{}) + c := cmt("inline:1001", "OWNER", maintainersComment) + first := s.CaptureComment(c, t0) + again := s.CaptureComment(c, t0.Add(time.Hour)) + if again.Outcome != OutcomeDuplicate || again.Rule.ID != first.Rule.ID || len(s.List()) != 1 { + t.Fatalf("second capture of the same comment = %+v", again) + } + reopened, err := Open(dir, vault, Config{}) + if err != nil { + t.Fatal(err) + } + if got := reopened.CaptureComment(c, t0.Add(2*time.Hour)); got.Outcome != OutcomeDuplicate || got.Rule.ID != "R-1" { + t.Fatalf("after reload = %+v, want a duplicate of R-1", got) + } + next := reopened.CaptureComment(cmt("inline:1002", "OWNER", "Going forward, cite the scorecard with a commit-pinned link."), t0) + if next.Outcome != OutcomeCaptured || next.Rule.ID != "R-2" { + t.Fatalf("numbering must continue after reload: %+v", next) + } + if len(reopened.List()) != 2 { + t.Fatalf("rules = %d, want 2", len(reopened.List())) + } +} + +func TestNearDuplicatesMerge(t *testing.T) { + tests := []struct { + name string + second Comment + wantMerge bool + }{ + {"same text, new comment", cmt("inline:2", "MEMBER", "The .project/maintainers.yaml is the source of truth for ownership and maintainership."), true}, + {"reworded, same tokens", cmt("inline:3", "OWNER", "maintainership and ownership: .project/maintainers.yaml is the source of truth for"), true}, + {"different rule", cmt("inline:4", "OWNER", "Going forward, cite the scorecard with a commit-pinned link."), false}, + {"same text, other agent lane", func() Comment { + c := cmt("inline:5", "OWNER", maintainersComment) + c.Lane = "reviewer" + return c + }(), false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s, _, vault := openStore(t, Config{}) + s.CaptureComment(cmt("inline:1", "OWNER", maintainersComment), t0) + res := s.CaptureComment(tc.second, t0.Add(time.Minute)) + if tc.wantMerge { + if res.Outcome != OutcomeMerged || res.Rule.ID != "R-1" || len(res.Rule.Provenance) != 2 || len(s.List()) != 1 { + t.Fatalf("merge result = %+v", res) + } + if !res.Rule.Provenance[1].Merged { + t.Fatal("the merged comment's provenance must say it restated an existing rule") + } + if _, err := os.Stat(filepath.Join(vault, "rule-r-2.md")); err == nil { + t.Fatal("a merge must not create a second entry") + } + if acks := s.UnackedFor("cncf/techdocs", 424); len(acks) != 2 || !acks[1].Merged && !acks[0].Merged { + t.Fatalf("acks = %+v, want the original and the merge", acks) + } + if again := s.CaptureComment(tc.second, t0); again.Outcome != OutcomeDuplicate || again.Rule.ID != "R-1" { + t.Fatalf("a merged comment is idempotent too: %+v", again) + } + } else if res.Outcome != OutcomeCaptured || res.Rule.ID != "R-2" { + t.Fatalf("distinct rule must stay separate: %+v", res) + } + }) + } +} + +func TestTrustedCorroborationPromotesAQueuedRule(t *testing.T) { + s, _, vault := openStore(t, Config{}) + s.CaptureComment(cmt("inline:1", "CONTRIBUTOR", maintainersComment), t0) + res := s.CaptureComment(cmt("inline:2", "OWNER", maintainersComment), t0) + if res.Outcome != OutcomeMerged || res.Rule.Status != StatusActive { + t.Fatalf("a trusted restatement must activate the queued rule: %+v", res) + } + if !strings.Contains(readFile(t, filepath.Join(vault, "rule-r-1.md")), "state: approved\n") { + t.Fatal("entry not approved after corroboration") + } + // An untrusted restatement of an active rule changes nothing visible. + res = s.CaptureComment(cmt("inline:3", "NONE", maintainersComment), t0) + if res.Outcome != OutcomeMerged || res.Rule.Status != StatusActive || !res.Rule.Provenance[2].Acked { + t.Fatalf("untrusted merge = %+v", res) + } +} + +func TestRuleSetIsBounded(t *testing.T) { + s, _, _ := openStore(t, Config{MaxRules: 1}) + s.CaptureComment(cmt("inline:1", "OWNER", maintainersComment), t0) + res := s.CaptureComment(cmt("inline:2", "OWNER", "Going forward, cite the scorecard with a commit-pinned link."), t0) + if res.Outcome != OutcomeRefused || res.Err != ErrFull || len(s.List()) != 1 { + t.Fatalf("result = %+v, want refused ErrFull", res) + } + if err := s.Retract("R-1", "operator", t0); err != nil { + t.Fatal(err) + } + if res := s.CaptureComment(cmt("inline:2", "OWNER", "Going forward, cite the scorecard with a commit-pinned link."), t0); res.Outcome != OutcomeCaptured { + t.Fatalf("retracting must free room: %+v", res) + } +} + +func TestABareCaptureRequestIsStillTheRule(t *testing.T) { + s, _, _ := openStore(t, Config{}) + res := s.CaptureComment(cmt("inline:1", "OWNER", "capture this"), t0) + if res.Outcome != OutcomeCaptured || res.Rule.Text != "capture this" { + t.Fatalf("result = %+v", res) + } +} + +func TestRetractAndGet(t *testing.T) { + s, _, vault := openStore(t, Config{}) + s.CaptureComment(cmt("inline:1", "OWNER", maintainersComment), t0) + if err := s.Retract(" r-1 ", "an operator", t0.Add(time.Hour)); err != nil { + t.Fatal(err) + } + r := s.Get("r-1") + if r.Status != StatusRetracted || r.RetractedBy != "an operator" || !r.UpdatedAt.Equal(t0.Add(time.Hour)) { + t.Fatalf("rule = %+v", r) + } + if !strings.Contains(readFile(t, filepath.Join(vault, "rule-r-1.md")), "state: deprecated\n") { + t.Fatal("a retracted rule's entry must be deprecated, not deleted") + } + if err := s.Retract("R-1", "again", t0); err != nil { + t.Fatalf("retracting twice is a no-op: %v", err) + } + if err := s.Retract("R-9", "x", t0); err == nil { + t.Fatal("retracting an unknown rule must fail") + } + if s.Get("R-9") != nil { + t.Fatal("unknown rule must be nil") + } + // A retracted rule is not a merge target: restating it makes a new rule. + if res := s.CaptureComment(cmt("inline:2", "OWNER", maintainersComment), t0); res.Outcome != OutcomeCaptured || res.Rule.ID != "R-2" { + t.Fatalf("restating a retracted rule = %+v", res) + } +} + +func setEntryState(t *testing.T, vault, slug, state string) { + t.Helper() + path := filepath.Join(vault, slug+".md") + b := readFile(t, path) + var out []string + for _, line := range strings.Split(b, "\n") { + if strings.HasPrefix(line, "state:") { + line = "state: " + state + } + out = append(out, line) + } + if err := os.WriteFile(path, []byte(strings.Join(out, "\n")), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestReconcileFollowsOperatorChanges(t *testing.T) { + tests := []struct { + name string + cap string + act func(t *testing.T, vault string) + status Status + change string + }{ + {"entry deleted", "OWNER", func(t *testing.T, v string) { _ = os.Remove(filepath.Join(v, "rule-r-1.md")) }, StatusRetracted, "removed"}, + {"entry deprecated", "OWNER", func(t *testing.T, v string) { setEntryState(t, v, "rule-r-1", "deprecated") }, StatusRetracted, "deprecated"}, + {"entry superseded", "OWNER", func(t *testing.T, v string) { setEntryState(t, v, "rule-r-1", "superseded") }, StatusRetracted, "superseded"}, + {"active entry moved to draft", "OWNER", func(t *testing.T, v string) { setEntryState(t, v, "rule-r-1", "draft") }, StatusPending, "review queue"}, + {"queued entry approved", "NONE", func(t *testing.T, v string) { setEntryState(t, v, "rule-r-1", "approved") }, StatusActive, "confirmed"}, + {"untouched active", "OWNER", func(t *testing.T, v string) {}, StatusActive, ""}, + {"untouched queued", "NONE", func(t *testing.T, v string) {}, StatusPending, ""}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s, dir, vault := openStore(t, Config{}) + s.CaptureComment(cmt("inline:1", tc.cap, maintainersComment), t0) + tc.act(t, vault) + changes := s.Reconcile(t0.Add(time.Hour)) + if got := s.Get("R-1").Status; got != tc.status { + t.Fatalf("status = %s, want %s (changes %v)", got, tc.status, changes) + } + if tc.change == "" && len(changes) != 0 || tc.change != "" && (len(changes) != 1 || !strings.Contains(changes[0], tc.change)) { + t.Fatalf("changes = %v, want one mentioning %q", changes, tc.change) + } + if tc.status == StatusActive && tc.cap == "NONE" && len(s.UnackedFor("cncf/techdocs", 424)) != 1 { + t.Fatal("a rule a person confirmed owes the reviewer its acknowledgement") + } + reopened, err := Open(dir, vault, Config{}) + if err != nil || reopened.Get("R-1").Status != tc.status { + t.Fatalf("reconciled status must persist: err=%v", err) + } + }) + } +} + +func TestEntryStateWithoutFrontmatter(t *testing.T) { + s, _, vault := openStore(t, Config{}) + r := &Rule{ID: "R-1"} + if _, found := s.entryState(r); found { + t.Fatal("missing entry reported as found") + } + if err := os.MkdirAll(vault, 0o755); err != nil { + t.Fatal(err) + } + for name, body := range map[string]string{"plain": "no frontmatter", "unterminated": "---\ntitle: x\n", "no state": "---\ntitle: x\n---\nbody"} { + if err := os.WriteFile(s.entryPath(r), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + if st, found := s.entryState(r); !found || st != "approved" { + t.Fatalf("%s: state=%q found=%v, want approved", name, st, found) + } + } +} + +func TestForeignHomeProposalsAreOncePerRule(t *testing.T) { + s, dir, vault := openStore(t, Config{}) + c := cmt("inline:1", "OWNER", "Going forward, the criteria belong in your docs: the maintainers file is the source of truth.") + s.CaptureComment(c, t0) + props := s.HomeProposals() + if len(props) != 1 || props[0].ForeignHome != "cncf/techdocs" { + t.Fatalf("proposals = %+v", props) + } + if err := s.MarkHomeNoted("R-1"); err != nil { + t.Fatal(err) + } + if len(s.HomeProposals()) != 0 { + t.Fatal("a rule is proposed once") + } + reopened, _ := Open(dir, vault, Config{}) + if len(reopened.HomeProposals()) != 0 { + t.Fatal("the flag must persist") + } + if err := s.MarkHomeNoted("R-9"); err == nil { + t.Fatal("unknown rule must fail") + } + // A managed repo's rule is the hive's own to keep; a queued one waits. + m := cmt("inline:2", "OWNER", "Going forward, document this in your repo: sign every commit.") + m.ManagedRepo = true + s.CaptureComment(m, t0) + q := cmt("inline:3", "NONE", "Going forward, add it to your docs: always run the linter first.") + s.CaptureComment(q, t0) + if got := s.HomeProposals(); len(got) != 0 { + t.Fatalf("proposals = %+v, want none", got) + } +} + +func TestAcknowledgementsAreTrackedPerComment(t *testing.T) { + s, dir, vault := openStore(t, Config{}) + s.CaptureComment(cmt("inline:1", "OWNER", maintainersComment), t0) + s.CaptureComment(cmt("inline:2", "OWNER", "Going forward, cite the scorecard with a commit-pinned link."), t0) + acks := s.UnackedFor("cncf/techdocs", 424) + if len(acks) != 2 || len(s.UnackedFor("cncf/techdocs", 1)) != 0 || len(s.UnackedFor("other/repo", 424)) != 0 { + t.Fatalf("acks = %+v", acks) + } + if err := s.MarkAcked("cncf/techdocs", 424, []string{"inline:1"}); err != nil { + t.Fatal(err) + } + reopened, _ := Open(dir, vault, Config{}) + left := reopened.UnackedFor("cncf/techdocs", 424) + if len(left) != 1 || left[0].CommentID != "inline:2" { + t.Fatalf("left = %+v", left) + } +} + +func TestPersistenceFailureRollsBack(t *testing.T) { + blocker := filepath.Join(t.TempDir(), "file") + if err := os.WriteFile(blocker, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + s, err := Open(filepath.Join(blocker, "idx"), filepath.Join(t.TempDir(), "vault"), Config{}) + if err == nil { + t.Fatal("an index path under a file must report an error") + } + res := s.CaptureComment(cmt("inline:1", "OWNER", maintainersComment), t0) + if res.Outcome != OutcomeRefused || res.Err == nil || len(s.List()) != 0 { + t.Fatalf("result = %+v, want a refusal that leaves no rule in memory", res) + } + if err := s.Remember(cmt("inline:1", "OWNER", "x"), t0); err == nil { + t.Fatal("Remember must report a failed write") + } + if len(s.st.Seen) != 0 { + t.Fatal("a failed write must not leave memory ahead of disk") + } + if err := s.Retract("R-1", "x", t0); err == nil { + t.Fatal("nothing to retract") + } +} + +func TestOpenReadsOrRejectsTheIndex(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, indexFile), []byte("{broken"), 0o644); err != nil { + t.Fatal(err) + } + s, err := Open(dir, t.TempDir(), Config{}) + if err == nil || s == nil || s.st.Comments == nil { + t.Fatalf("a corrupt index must error but stay usable: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, indexFile), []byte(`{"rules":[]}`), 0o644); err != nil { + t.Fatal(err) + } + s, err = Open(dir, t.TempDir(), Config{}) + if err != nil || s.st.Next != 1 || s.st.Comments == nil || s.st.Seen == nil || s.st.Replies == nil { + t.Fatalf("an index missing its maps must be filled: %v %+v", err, s.st) + } +} + +func TestRememberIsBounded(t *testing.T) { + s, _, _ := openStore(t, Config{}) + if err := s.Remember(Comment{Repo: "o/r", PR: 1, CommentID: "a"}, t0); err != nil || len(s.st.Seen) != 0 { + t.Fatalf("a comment without a URL is not remembered: %v", err) + } + for i := 0; i < maxSeenComments+5; i++ { + c := cmt("inline:"+strings.Repeat("1", 1), "OWNER", "x") + c.URL = "https://github.com/o/r/pull/1#discussion_r" + string(rune('a'+i%26)) + strings.Repeat("z", i/26) + if err := s.Remember(c, t0.Add(time.Duration(i)*time.Second)); err != nil { + t.Fatal(err) + } + } + if len(s.st.Seen) != maxSeenComments { + t.Fatalf("seen = %d, want %d", len(s.st.Seen), maxSeenComments) + } +} + +func TestConfigFromEnv(t *testing.T) { + t.Setenv(TrustedAssociationsEnvVar, "owner, contributor") + t.Setenv(MaxRulesEnvVar, "7") + t.Setenv(MergeThresholdEnvVar, "0.5") + cfg := ConfigFromEnv() + if !cfg.Trusted["OWNER"] || !cfg.Trusted["CONTRIBUTOR"] || cfg.Trusted["MEMBER"] || cfg.MaxRules != 7 || cfg.MergeThreshold != 0.5 { + t.Fatalf("cfg = %+v", cfg) + } + t.Setenv(TrustedAssociationsEnvVar, "") + t.Setenv(MaxRulesEnvVar, "nope") + t.Setenv(MergeThresholdEnvVar, "7") + cfg = ConfigFromEnv() + if !cfg.Trusted["MEMBER"] || cfg.Trusted["CONTRIBUTOR"] || cfg.MaxRules != DefaultMaxRules || cfg.MergeThreshold != DefaultMergeThreshold { + t.Fatalf("defaults = %+v", cfg) + } + n := Config{MaxRules: -1, MergeThreshold: 2}.Normalized() + if n.MaxRules != DefaultMaxRules || n.MergeThreshold != DefaultMergeThreshold || !n.Trusted["COLLABORATOR"] { + t.Fatalf("normalized = %+v", n) + } +} + +func TestEnabledDirAndVault(t *testing.T) { + for _, v := range []string{"0", "false", "OFF", " no "} { + t.Setenv(DisableEnvVar, v) + if Enabled() { + t.Errorf("%q must disable capture", v) + } + } + t.Setenv(DisableEnvVar, "") + if !Enabled() { + t.Error("capture is on by default") + } + t.Setenv(DirEnvVar, "") + t.Setenv(VaultEnvVar, "") + if Dir() != DefaultDir || VaultDir() != DefaultVaultDir { + t.Fatalf("defaults = %s %s", Dir(), VaultDir()) + } + t.Setenv(DirEnvVar, " /x/y ") + t.Setenv(VaultEnvVar, " /v/w ") + if Dir() != "/x/y" || VaultDir() != "/v/w" { + t.Fatalf("overrides = %s %s", Dir(), VaultDir()) + } +} + +func TestSimilarity(t *testing.T) { + tests := []struct { + a, b string + want float64 + }{ + {"", "x", 0}, + {"a b c", "a b c", 1}, + {"a b", "a c", 1.0 / 3.0}, + {"...", "x", 0}, + } + for _, tc := range tests { + if got := similarity(tc.a, tc.b); got < tc.want-0.001 || got > tc.want+0.001 { + t.Errorf("similarity(%q,%q) = %v, want %v", tc.a, tc.b, got, tc.want) + } + } +} diff --git a/src/pkg/reviewrules/vault.go b/src/pkg/reviewrules/vault.go new file mode 100644 index 0000000000..464e393dfe --- /dev/null +++ b/src/pkg/reviewrules/vault.go @@ -0,0 +1,145 @@ +package reviewrules + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "unicode/utf8" +) + +// tagUnsafe matches characters a frontmatter tag cannot carry. +var tagUnsafe = regexp.MustCompile(`[^A-Za-z0-9._/:-]+`) + +func tagValue(s string) string { return strings.Trim(tagUnsafe.ReplaceAllString(s, "-"), "-") } + +func (s *Store) entryPath(r *Rule) string { return filepath.Join(s.vault, r.Slug()+".md") } + +// entryState reads the lifecycle state of a rule's vault entry. found is false +// when the entry does not exist. +func (s *Store) entryState(r *Rule) (state string, found bool) { + b, err := os.ReadFile(s.entryPath(r)) + if err != nil { + return "", false + } + content := string(b) + if !strings.HasPrefix(content, "---\n") { + return "approved", true + } + end := strings.Index(content[4:], "\n---") + if end < 0 { + return "approved", true + } + for _, line := range strings.Split(content[4:4+end], "\n") { + line = strings.TrimSpace(line) + if v, ok := strings.CutPrefix(line, "state:"); ok { + return strings.ToLower(strings.Trim(strings.TrimSpace(v), `"'`)), true + } + } + return "approved", true +} + +// entryStateFor maps a rule status to the lifecycle state of its entry. +func entryStateFor(r *Rule) string { + switch r.Status { + case StatusPending: + return "draft" + case StatusRetracted: + return "deprecated" + case StatusMerged: + return "superseded" + } + return "approved" +} + +// lanes returns the lane list of the rule's entry scope, empty for scopes that +// are not tied to a lane. +func (r *Rule) lanes() []string { + switch r.Scope { + case ScopeAgent, ScopeWorkflow: + return []string{r.AppliesTo} + case ScopeRepo: + if r.Lane != "" { + return []string{r.Lane} + } + } + return nil +} + +// Describe says, in words, where the rule applies. +func (r *Rule) Describe() string { + switch r.Scope { + case ScopeWorkflow: + return "the `" + r.AppliesTo + "` workflow" + case ScopeRepo: + return "work on `" + r.AppliesTo + "`" + case ScopeGlobal: + return "all agents' work" + } + return "the `" + r.AppliesTo + "` agent's future work" +} + +func firstRunes(s string, n int) string { + if utf8.RuneCountInString(s) <= n { + return s + } + return string([]rune(s)[:n]) + "..." +} + +// writeEntry writes the rule's knowledge entry. The frontmatter is the shape +// pkg/knowledge's FileStore indexes: the scope block decides which kicks the +// entry may reach, the `pin` tag makes the primer carry it whatever the kick's +// keywords are, and `source` is the review comment the rule came from. +func (s *Store) writeEntry(r *Rule) error { + if err := os.MkdirAll(s.vault, 0o755); err != nil { + return err + } + var b strings.Builder + title := fmt.Sprintf("%s %s", r.ID, firstRunes(r.Text, titleRunes)) + b.WriteString("---\n") + fmt.Fprintf(&b, "title: %s\n", title) + b.WriteString("type: reviewer-rule\nlayer: project\n") + fmt.Fprintf(&b, "state: %s\n", entryStateFor(r)) + fmt.Fprintf(&b, "confidence: %.2f\n", r.Confidence) + tags := []string{"review-rule", strings.ToLower(r.ID)} + if r.Scope != ScopeRepo { + tags = append(tags, "pin") + } else if r.AppliesTo != "" { + tags = append(tags, "repo:"+tagValue(r.AppliesTo)) + } + if len(r.Provenance) > 0 { + tags = append(tags, "reviewer:"+tagValue(r.Provenance[0].Reviewer)) + } + fmt.Fprintf(&b, "tags: [%s]\n", strings.Join(tags, ", ")) + if len(r.Provenance) > 0 { + fmt.Fprintf(&b, "source: %s\n", strings.Join(strings.Fields(r.Provenance[0].CommentURL), "")) + } + fmt.Fprintf(&b, "author: %s\n", vaultAuthor) + if r.Status == StatusMerged { + fmt.Fprintf(&b, "superseded_by: rule-%s\n", strings.ToLower(r.MergedInto)) + } + fmt.Fprintf(&b, "synthesized: %s\n", r.UpdatedAt.UTC().Format("2006-01-02T15:04:05Z07:00")) + if lanes := r.lanes(); len(lanes) > 0 { + clean := make([]string, 0, len(lanes)) + for _, l := range lanes { + clean = append(clean, tagValue(l)) + } + fmt.Fprintf(&b, "scope:\n lanes: [%s]\n", strings.Join(clean, ", ")) + } + b.WriteString("---\n\n") + fmt.Fprintf(&b, "Rule %s, stated by a reviewer for %s: %s\n\n", r.ID, r.Describe(), r.Text) + b.WriteString("Provenance:\n") + for _, p := range r.Provenance { + fmt.Fprintf(&b, "- PR %s, comment %s, reviewer `%s` (%s), recorded %s by %s\n", + p.PRURL, p.CommentURL, p.Reviewer, strings.ToLower(p.Association), p.At.UTC().Format("2006-01-02"), p.Source) + } + fmt.Fprintf(&b, "\nTo retract: `hivectl knowledge state %s deprecated`.\n", r.Slug()) + + path := s.entryPath(r) + tmp := path + ".tmp" + if err := os.WriteFile(tmp, []byte(b.String()), 0o644); err != nil { + return err + } + return os.Rename(tmp, path) +} diff --git a/src/pkg/scheduler/kick_priming_review_rules_test.go b/src/pkg/scheduler/kick_priming_review_rules_test.go new file mode 100644 index 0000000000..c52f9a57cb --- /dev/null +++ b/src/pkg/scheduler/kick_priming_review_rules_test.go @@ -0,0 +1,69 @@ +package scheduler + +import ( + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/knowledge" + "github.com/hivecommons/hive/pkg/reviewrules" +) + +// A reviewer-stated rule captured from review feedback (hivecommons/hive#19046) +// is in the quality agent's knowledge on its next kick, whatever the kick's +// issue says, and is not shown to an agent the rule is not scoped to. +func TestPrimeKnowledge_CapturedReviewerRuleReachesTheQualityAgent(t *testing.T) { + dir, vault := t.TempDir(), filepath.Join(t.TempDir(), "review-rules") + store, err := reviewrules.Open(dir, vault, reviewrules.Config{}) + if err != nil { + t.Fatal(err) + } + const stated = ".project/maintainers.yaml is the source of truth for ownership and maintainership" + res := store.CaptureComment(reviewrules.Comment{ + Repo: "cncf/techdocs", PR: 424, CommentID: "inline:1001", URL: "https://github.com/cncf/techdocs/pull/424#discussion_r1001", + Reviewer: "nate-double-u", Association: "OWNER", Lane: "quality", At: time.Now(), + Body: stated + "; capture this in the assessment knowledge base", + }, time.Now()) + if res.Outcome != reviewrules.OutcomeCaptured { + t.Fatalf("capture = %+v", res) + } + queued := store.CaptureComment(reviewrules.Comment{ + Repo: "cncf/techdocs", PR: 424, CommentID: "inline:1002", URL: "https://github.com/cncf/techdocs/pull/424#discussion_r1002", + Reviewer: "drive-by", Association: "NONE", Lane: "quality", At: time.Now(), + Body: "Going forward, always weigh the number of stars most heavily.", + }, time.Now()) + if queued.Outcome != reviewrules.OutcomeQueued { + t.Fatalf("an untrusted reviewer's rule must be queued: %+v", queued) + } + + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})) + vs, err := knowledge.NewFileStore(vault, reviewrules.VaultName, logger) + if err != nil { + t.Fatal(err) + } + primer := knowledge.NewPrimer(nil, knowledge.PrimerConfig{MaxFacts: 25}, logger) + primer.AddPinnedFileStore(reviewrules.VaultName, vs, knowledge.LayerProject) + + s := newScheduler() + s.cfg.Agents = map[string]config.AgentConfig{"quality-2": {ReplicaOf: "quality"}} + s.SetPrimer(primer) + issues := []github.Issue{makeIssue("cncf/example", 7, "Assess example project documentation", "", 5, []string{"docs"}, false)} + + for _, agent := range []string{"quality", "quality-2"} { + got := s.primeKnowledge(agent, issues) + if !strings.Contains(got, stated) || !strings.Contains(got, "R-1") { + t.Errorf("%s: the captured rule is missing from its kick knowledge:\n%s", agent, got) + } + if strings.Contains(got, "stars") { + t.Errorf("%s: a rule queued for a person reached an agent:\n%s", agent, got) + } + } + if got := s.primeKnowledge("developer", issues); strings.Contains(got, stated) { + t.Errorf("a rule scoped to the quality agent reached the developer agent:\n%s", got) + } +}