From 959333223159efcf5a8deadb45983da59a1f8514 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:20:56 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Review-rule=20capture=20reads=20its?= =?UTF-8?q?=20own=20candidate=20set?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rule capture only saw the routed burst, so a rule stated in a comment that was already handled, sat in a resolved or answered thread, or was a plain PR comment not addressed to the hive was never classified. Capture now reads every human comment on the PR (RuleCandidates), and the rule index keeps its own evaluated comment ids, separate from the sweep's handled ids, so the first pass is a one-time backfill and later passes read only new comments. Kick routing is unchanged; the brief looks up rules by comment id. Closes #19136 Co-Authored-By: Claude Opus 5.5 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-19136-review-rule-candidates.md | 1 + .../hive/review_feedback_rules_wire_test.go | 66 +++++++++++++++ src/cmd/hive/review_feedback_wire.go | 55 ++++++++---- src/docs/review-feedback-rules.md | 10 ++- src/pkg/github/pr_feedback.go | 84 +++++++++++++++---- src/pkg/github/pr_feedback_test.go | 43 ++++++++++ src/pkg/reviewrules/store.go | 66 ++++++++++++++- src/pkg/reviewrules/store_test.go | 51 +++++++++++ 8 files changed, 341 insertions(+), 35 deletions(-) create mode 100644 changelog.d/fixed-19136-review-rule-candidates.md diff --git a/changelog.d/fixed-19136-review-rule-candidates.md b/changelog.d/fixed-19136-review-rule-candidates.md new file mode 100644 index 0000000000..9c2e66d065 --- /dev/null +++ b/changelog.d/fixed-19136-review-rule-candidates.md @@ -0,0 +1 @@ +- Reviewer-stated rule capture now reads every human comment on the hive's open PRs (inline, review body and plain conversation comments), including ones already handled, in resolved threads or not addressed to the hive, and the rule index tracks which comments it has evaluated so the first pass after upgrade backfills once and later passes classify only new comments ([#19136](https://github.com/hivecommons/hive/issues/19136)). diff --git a/src/cmd/hive/review_feedback_rules_wire_test.go b/src/cmd/hive/review_feedback_rules_wire_test.go index 532586eda0..ea09df5a68 100644 --- a/src/cmd/hive/review_feedback_rules_wire_test.go +++ b/src/cmd/hive/review_feedback_rules_wire_test.go @@ -51,6 +51,7 @@ func (h *ruleHarness) sweep(t *testing.T) reviewFeedbackSweep { func (h *ruleHarness) feedback(items ...github.FeedbackItem) { fb := openDraftFeedback() fb.Items = items + fb.RuleCandidates = items h.src.byKey[feedbackTestKey] = fb } @@ -386,3 +387,68 @@ func TestReviewFeedbackSweep_ReplyMustNameCapturedRule(t *testing.T) { } }) } + +// The maintainers.yaml comment on cncf/techdocs#424 was already named in a +// handled marker, so routing no longer returns it; a plain conversation comment +// does not address the hive. Rule capture reads its own candidate set, so both +// are captured on the first pass (the backfill) without a kick, and a later +// pass does not read them again (hivecommons/hive#19136). +func TestReviewFeedbackSweep_RuleCaptureReadsHandledAndUnaddressedComments(t *testing.T) { + h := newRuleHarness(t, reviewrules.Config{}) + plain := ruleItem("conversation:2002", "OWNER", "Going forward, cite the scorecard with a commit-pinned link.") + plain.Kind, plain.ThreadID, plain.Path, plain.Line = github.PRCommentConversation, "", "", 0 + fb := openDraftFeedback() + fb.Items = nil + fb.RuleCandidates = []github.FeedbackItem{ruleItem("inline:1001", "MEMBER", maintainersFixture), plain} + h.src.byKey[feedbackTestKey] = fb + res := h.sweep(t).run(context.Background(), parseReviewFeedbackKey) + if res.RulesCaptured != 2 || res.Kicked != 0 || len(h.rules.List()) != 2 { + t.Fatalf("res = %+v rules = %d, want both captured and no kick", res, len(h.rules.List())) + } + if r := h.rules.Get("R-1"); r == nil || r.Text != maintainersRuleText { + t.Fatalf("R-1 = %+v", r) + } + for _, id := range []string{"inline:1001", "conversation:2002"} { + if !h.rules.Evaluated("cncf/techdocs", 424, id) { + t.Fatalf("%s not recorded as evaluated", id) + } + } + if len(h.src.comments) != 1 || !strings.Contains(h.src.comments[0], "R-1") || !strings.Contains(h.src.comments[0], "R-2") { + t.Fatalf("comments = %q, want one acknowledgement naming both rules", h.src.comments) + } + + // Later passes classify only new comments. + fb.RuleCandidates = append(fb.RuleCandidates, ruleItem("inline:1003", "OWNER", "The score is too high.")) + h.src.byKey[feedbackTestKey] = fb + h.now = h.now.Add(time.Hour) + res = h.sweep(t).run(context.Background(), parseReviewFeedbackKey) + if res.RulesCaptured+res.RulesMerged+res.RulesQueued != 0 || len(h.src.comments) != 1 || !h.rules.Evaluated("cncf/techdocs", 424, "inline:1003") { + t.Fatalf("res = %+v comments = %d", res, len(h.src.comments)) + } +} + +// A candidate still inside the debounce may be edited; it is read once it is +// quiet, and a kick of a comment captured on an earlier pass still names the rule. +func TestReviewFeedbackSweep_RuleCaptureWaitsForDebounceAndBriefNamesEarlierCapture(t *testing.T) { + h := newRuleHarness(t, reviewrules.Config{}) + it := ruleItem("inline:1001", "OWNER", maintainersFixture) + it.CreatedAt = h.now + fb := openDraftFeedback() + fb.Items = nil + fb.RuleCandidates = []github.FeedbackItem{it} + h.src.byKey[feedbackTestKey] = fb + if res := h.sweep(t).run(context.Background(), parseReviewFeedbackKey); res.RulesCaptured != 0 || h.rules.Evaluated("cncf/techdocs", 424, "inline:1001") { + t.Fatalf("a comment still in the debounce was read: %+v", res) + } + h.now = h.now.Add(time.Hour) + if res := h.sweep(t).run(context.Background(), parseReviewFeedbackKey); res.RulesCaptured != 1 { + t.Fatalf("res = %+v", res) + } + fb.Items = []github.FeedbackItem{it} + h.src.byKey[feedbackTestKey] = fb + h.now = h.now.Add(time.Hour) + res := h.sweep(t).run(context.Background(), parseReviewFeedbackKey) + if res.Kicked != 1 || res.RulesCaptured != 0 || !strings.Contains(h.kick.kicks["quality"][0], "R-1 (recorded") { + t.Fatalf("res = %+v kicks = %q", res, h.kick.kicks) + } +} diff --git a/src/cmd/hive/review_feedback_wire.go b/src/cmd/hive/review_feedback_wire.go index d6e804a49c..769ce48340 100644 --- a/src/cmd/hive/review_feedback_wire.go +++ b/src/cmd/hive/review_feedback_wire.go @@ -124,6 +124,7 @@ func (s reviewFeedbackSweep) run(ctx context.Context, parse func(key string) (re s.applyRuleReplies(repo, number, fb.HiveReplies) s.postRuleNotices(ctx, repo, number) s.checkRuleReply(ctx, key, repo, number, fb.AgentReplies, &res) + s.captureRules(ctx, repo, number, redPRFixOwner(s.Owners[key]), fb.RuleCandidates, &res) } if !fb.Open() || len(fb.Items) == 0 { continue @@ -135,10 +136,9 @@ func (s reviewFeedbackSweep) run(ctx context.Context, parse func(key string) (re s.Logger.Debug("review feedback: burst still arriving, waiting to coalesce", "repo", repo, "pr", number, "comments", len(d.Items)) case reviewfeedback.VerdictCapReached: res.CapReached++ - s.captureRules(ctx, repo, number, redPRFixOwner(s.Owners[key]), d.Items, &res) s.needsHuman(ctx, key, repo, number, "it has been routed to its agent "+strconv.Itoa(s.Limits.WindowCap)+" times in "+s.Limits.Window.String()+"; further feedback is waiting, and a new comment from a person resets this", d.Items, &res) case reviewfeedback.VerdictKick: - recorded := s.captureRules(ctx, repo, number, redPRFixOwner(s.Owners[key]), d.Items, &res) + recorded := s.recordedRules(repo, number, d.Items) if route.Agent == "" { s.needsHuman(ctx, key, repo, number, route.Reason, d.Items, &res) continue @@ -189,28 +189,38 @@ func (s reviewFeedbackSweep) needsHuman(ctx context.Context, key, repo string, n res.NeedsHuman++ } -// captureRules is the deterministic half of rule capture (#19046): every -// comment in the burst that states a rule is written to the hive's knowledge -// store whether or not the agent later records it, idempotently per comment. -// It runs for every verdict that carries comments, so a rule is not lost to the -// cap or to an unavailable agent. It returns the rules for the agent's brief. -func (s reviewFeedbackSweep) captureRules(ctx context.Context, repo string, number int, owner string, items []github.FeedbackItem, res *reviewFeedbackResult) []reviewfeedback.RecordedRule { +// captureRules is the deterministic half of rule capture (#19046): every human +// comment on the PR that states a rule is written to the hive's knowledge store +// whether or not the agent later records it, idempotently per comment. It reads +// its own candidate set (#19136), not the routed burst, so a comment that was +// already handled, sits in a resolved thread, or does not address the hive is +// still classified. Each comment is classified once: the rule index keeps the +// evaluated ids, so the first pass over an index without them is a one-time +// backfill and later passes read only new comments. A comment newer than the +// debounce may still be edited and waits for a later pass. +func (s reviewFeedbackSweep) captureRules(ctx context.Context, repo string, number int, owner string, candidates []github.FeedbackItem, res *reviewFeedbackResult) { if s.Rules == nil { - return nil + return } lane := owner if s.Lane != nil { lane = s.Lane(owner) } managed := s.Managed != nil && s.Managed(repo) - var recorded []reviewfeedback.RecordedRule - for _, it := range items { + var evaluated []string + for _, it := range candidates { + if s.Now.Sub(it.CreatedAt) < s.Limits.Debounce || s.Rules.Evaluated(repo, number, it.ID) { + continue + } c := reviewrules.Comment{Repo: repo, PR: number, CommentID: it.ID, URL: it.URL, Reviewer: it.Author, Association: it.AuthorAssociation, Body: it.Body, At: it.CreatedAt, Lane: lane, ManagedRepo: managed} if err := s.Rules.Remember(c, s.Now); err != nil { s.Logger.Warn("review feedback: could not remember the comment for rule evidence", "repo", repo, "pr", number, "comment", it.ID, "error", err) } r := s.Rules.CaptureComment(c, s.Now) + if r.Outcome != reviewrules.OutcomeRefused { + evaluated = append(evaluated, it.ID) // a refused rule is tried again on a later pass + } switch r.Outcome { case reviewrules.OutcomeNotRule: continue @@ -229,13 +239,28 @@ func (s reviewFeedbackSweep) captureRules(ctx context.Context, repo string, numb "reviewer", it.Author, "association", it.AuthorAssociation, "outcome", string(r.Outcome), "rule", r.Rule.ID, "scope", string(r.Rule.Scope), "applies_to", r.Rule.AppliesTo, "signals", r.Signals) } - if r.Rule == nil || (r.Rule.Status != reviewrules.StatusActive && r.Rule.Status != reviewrules.StatusPending) { + } + if err := s.Rules.MarkEvaluated(repo, number, evaluated, s.Now); err != nil { + s.Logger.Warn("review feedback: could not persist the evaluated comment ids; a later pass reads them again", "repo", repo, "pr", number, "error", err) + } + s.postRuleNotices(ctx, repo, number) +} + +// recordedRules returns the live rules the routed comments were captured as, +// for the agent's brief, whichever pass captured them. +func (s reviewFeedbackSweep) recordedRules(repo string, number int, items []github.FeedbackItem) []reviewfeedback.RecordedRule { + if s.Rules == nil { + return nil + } + var recorded []reviewfeedback.RecordedRule + for _, it := range items { + r := s.Rules.RuleForComment(repo, number, it.ID) + if r == nil || (r.Status != reviewrules.StatusActive && r.Status != reviewrules.StatusPending) { continue } - recorded = append(recorded, reviewfeedback.RecordedRule{ID: r.Rule.ID, Text: r.Rule.Text, Applies: r.Rule.Describe(), - Queued: r.Rule.Status == reviewrules.StatusPending, SourceURL: it.URL}) + recorded = append(recorded, reviewfeedback.RecordedRule{ID: r.ID, Text: r.Text, Applies: r.Describe(), + Queued: r.Status == reviewrules.StatusPending, SourceURL: it.URL}) } - s.postRuleNotices(ctx, repo, number) return recorded } diff --git a/src/docs/review-feedback-rules.md b/src/docs/review-feedback-rules.md index ccea893249..adfeb1b6e0 100644 --- a/src/docs/review-feedback-rules.md +++ b/src/docs/review-feedback-rules.md @@ -8,7 +8,7 @@ leaving it to the agent's discretion ([#19046](https://github.com/hivecommons/hi ## What happens -1. **Classify.** Each comment in a coalesced burst is matched against fixed phrase signals. A comment with at least one is a rule; the rest +1. **Classify.** Each human comment on the PR is matched against fixed phrase signals. A comment with at least one is a rule; the rest are corrections and are handled as before. The signals are `source-of-truth`, `authoritative`, `always` ("should always", "must always"), `never` ("should never", "must never"), `going-forward` ("going forward", "from now on", "next time", "for future assessments"), `capture-request` ("capture this", "document this", "record it"), `assessment-criteria` ("assessment criteria", "assessment knowledge") @@ -31,8 +31,12 @@ leaving it to the agent's discretion ([#19046](https://github.com/hivecommons/hi so the quality agent receives its rules on its next kick; a rule scoped to a workflow reaches the lane or workflow model of that name. The usual lifecycle filter and `knowledge.agent_scopes` still apply, so drafts, retracted rules and out-of-scope agents never see a rule. -Capture happens when the burst is routed, so it also happens when the per-PR cap was reached or no agent could take the kick. It does not -change the sweep's debounce, cap or thread behavior. +Capture reads its own candidate set, not the routed burst: every human comment on the hive's open PRs in the attribution window (inline, +review body, conversation), whether or not it was already handled, sits in a resolved thread, or addresses the hive. A comment is read once +it has been quiet for the sweep's debounce. The rule index keeps the ids it has evaluated, separately from the sweep's handled ids, so the +first pass after an upgrade is a one-time backfill over the comments still visible on those PRs and later passes read only new comments. So +capture also happens when the per-PR cap was reached, no agent could take the kick, or no kick is due at all. It does not change the sweep's +debounce, cap, routing or thread behavior. ## The agent's structured block diff --git a/src/pkg/github/pr_feedback.go b/src/pkg/github/pr_feedback.go index 664f1df2d6..badd30e4a5 100644 --- a/src/pkg/github/pr_feedback.go +++ b/src/pkg/github/pr_feedback.go @@ -103,6 +103,10 @@ type PRFeedback struct { // AgentReplies are the hive's comments that carry the handled-ids // marker: the kicked agent's replies to feedback. AgentReplies []FeedbackMessage + // RuleCandidates are every human comment on the PR, whatever its handled, + // resolved or addressed state, oldest first. Rule capture reads these + // (hivecommons/hive#19136); kick routing reads Items. + RuleCandidates []FeedbackItem } // Open reports whether the PR can still take a revision. @@ -172,13 +176,14 @@ func (c *Client) FetchPRFeedback(ctx context.Context, repo string, number int) ( state = "closed" } return PRFeedback{ - State: state, - Draft: raw.IsDraft, - HeadRef: raw.HeadRefName, - Items: UnansweredFeedback(raw.rawPRConversation, c.isHumanFeedbackAuthor, c.isAgentCommentAuthor, c.addressesHive), - Source: source, - HiveReplies: HiveRuleReplies(raw.rawPRConversation, c.isHiveAuthor), - AgentReplies: HiveFeedbackReplies(raw.rawPRConversation, c.isHiveAuthor), + State: state, + Draft: raw.IsDraft, + HeadRef: raw.HeadRefName, + Items: UnansweredFeedback(raw.rawPRConversation, c.isHumanFeedbackAuthor, c.isAgentCommentAuthor, c.addressesHive), + Source: source, + HiveReplies: HiveRuleReplies(raw.rawPRConversation, c.isHiveAuthor), + AgentReplies: HiveFeedbackReplies(raw.rawPRConversation, c.isHiveAuthor), + RuleCandidates: RuleCandidates(raw.rawPRConversation, c.isHumanFeedbackAuthor), }, nil } @@ -252,15 +257,7 @@ func UnansweredFeedback(raw rawPRConversation, isHuman func(*rawPRCommentAuthor) } } } - human := func(rc rawPRComment) bool { - if rc.Author == nil || strings.TrimSpace(rc.Body) == "" || HasAttributionTrailer(rc.Body) { - return false - } - if !feedbackTrustedAssociations[strings.ToUpper(rc.AuthorAssociation)] { - return false - } - return isHuman != nil && isHuman(rc.Author) - } + human := func(rc rawPRComment) bool { return humanFeedback(rc, isHuman) } var out []FeedbackItem add := func(kind PRCommentKind, rc rawPRComment, thread, path string, line int, ctxThread []FeedbackMessage) { id := string(kind) + ":" + strconv.FormatInt(rc.DatabaseID, 10) @@ -321,6 +318,61 @@ func UnansweredFeedback(raw rawPRConversation, isHuman func(*rawPRCommentAuthor) return out } +// humanFeedback reports whether a comment is a person's: non-empty, from a +// human (isHuman) with a trusted association, and without a hive attribution +// trailer. +func humanFeedback(rc rawPRComment, isHuman func(*rawPRCommentAuthor) bool) bool { + if rc.Author == nil || strings.TrimSpace(rc.Body) == "" || HasAttributionTrailer(rc.Body) { + return false + } + if !feedbackTrustedAssociations[strings.ToUpper(rc.AuthorAssociation)] { + return false + } + return isHuman != nil && isHuman(rc.Author) +} + +// RuleCandidates returns every human comment on a PR for rule capture +// (hivecommons/hive#19136): inline comments, non-empty review bodies of any +// state, and conversation comments, with the same author checks as +// UnansweredFeedback but none of its routing gates. A comment named in a +// handled marker, older than the hive's reply in its thread, in a resolved +// thread, or a conversation comment not addressed to the hive is still a +// candidate, because a reviewer can state a rule in any of them. Which +// candidates were already classified is the rule index's concern. +func RuleCandidates(raw rawPRConversation, isHuman func(*rawPRCommentAuthor) bool) []FeedbackItem { + var out []FeedbackItem + add := func(kind PRCommentKind, rc rawPRComment, thread, path string, line int) { + if !humanFeedback(rc, isHuman) { + return + } + out = append(out, FeedbackItem{ID: string(kind) + ":" + strconv.FormatInt(rc.DatabaseID, 10), ThreadID: thread, Kind: kind, + Author: rc.Author.Login, Body: truncateCommentBody(rc.Body), Path: path, Line: line, URL: rc.URL, CreatedAt: rc.CreatedAt, + AuthorAssociation: strings.ToUpper(rc.AuthorAssociation)}) + } + for _, rc := range raw.Comments.Nodes { + add(PRCommentConversation, rc, "", "", 0) + } + for _, rc := range raw.Reviews.Nodes { + add(PRCommentReview, rc, "", "", 0) + } + for _, th := range raw.ReviewThreads.Nodes { + line := 0 + if th.Line != nil { + line = *th.Line + } + for _, rc := range th.Comments.Nodes { + add(PRCommentInline, rc, th.ID, th.Path, line) + } + } + sort.SliceStable(out, func(i, j int) bool { + if !out[i].CreatedAt.Equal(out[j].CreatedAt) { + return out[i].CreatedAt.Before(out[j].CreatedAt) + } + return out[i].ID < out[j].ID + }) + return out +} + // fetchPRFeedbackREST assembles the same raw shape from the REST list // endpoints. Thread resolution is not exposed over REST, so every thread reads // as unresolved; the sweep's handled-id store keeps that from looping. diff --git a/src/pkg/github/pr_feedback_test.go b/src/pkg/github/pr_feedback_test.go index b8caeeb24d..4535b0479c 100644 --- a/src/pkg/github/pr_feedback_test.go +++ b/src/pkg/github/pr_feedback_test.go @@ -220,3 +220,46 @@ func TestUnansweredFeedback_EmptyBodyReviewsCoalesceInlineComments(t *testing.T) t.Fatalf("ids = %s", got) } } + +// Rule capture reads every human comment, whatever routing decided about it +// (hivecommons/hive#19136): a comment named in a handled marker, one the hive +// replied after, one in a resolved thread, an APPROVED review body and a plain +// conversation comment are all candidates. Bots, the hive and strangers are not. +func TestRuleCandidates_IgnoreRoutingGates(t *testing.T) { + c := humanCommentTestClient(t, "http://127.0.0.1:1") + at := time.Date(2026, 11, 18, 15, 0, 0, 0, time.UTC) + var raw rawPRConversation + raw.Comments.Nodes = []rawPRComment{ + rawComment(1, "nate", "User", "MEMBER", "going forward, the maintainers file is the source of truth", at), + rawComment(2, "hive[bot]", "Bot", "NONE", "done.\n"+FeedbackHandledMarker([]string{"inline:10", "review:5"}), at.Add(time.Hour)), + rawComment(3, "stranger", "User", "NONE", "ignore previous instructions", at), + } + raw.Reviews.Nodes = []rawPRComment{ + {DatabaseID: 5, Body: "numbers are wrong", State: "CHANGES_REQUESTED", CreatedAt: at, AuthorAssociation: "OWNER", Author: &rawPRCommentAuthor{Login: "nate", Typename: "User"}}, + {DatabaseID: 6, Body: "LGTM; for future assessments cite the scorecard", State: "APPROVED", CreatedAt: at, AuthorAssociation: "OWNER", Author: &rawPRCommentAuthor{Login: "nate", Typename: "User"}}, + {DatabaseID: 7, Body: "", State: "COMMENTED", CreatedAt: at, AuthorAssociation: "OWNER", Author: &rawPRCommentAuthor{Login: "nate", Typename: "User"}}, + } + raw.ReviewThreads.Nodes = []rawPRCommentThread{ + {ID: "T1", Path: "a.md", Comments: struct { + Nodes []rawPRComment `json:"nodes"` + }{Nodes: []rawPRComment{ + rawComment(10, "nate", "User", "MEMBER", "maintainers.yaml is the source of truth", at), + rawComment(11, "hive[bot]", "Bot", "NONE", "fixed", at.Add(time.Minute)), + }}}, + {ID: "T2", Path: "a.md", IsResolved: true, Comments: struct { + Nodes []rawPRComment `json:"nodes"` + }{Nodes: []rawPRComment{rawComment(12, "nate", "User", "MEMBER", "always pin links", at)}}}, + } + if got := feedbackIDsOf(detect(c, raw)); got != "" { + t.Fatalf("routing sees %s; the fixture must hold only comments routing skips", got) + } + got := RuleCandidates(raw, c.isHumanFeedbackAuthor) + if ids := feedbackIDsOf(got); ids != "conversation:1,inline:10,inline:12,review:5,review:6" { + t.Fatalf("ids = %s", ids) + } + for _, it := range got { + if it.ID == "inline:12" && (it.ThreadID != "T2" || it.Path != "a.md" || it.AuthorAssociation != "MEMBER") { + t.Fatalf("inline candidate lost its context: %+v", it) + } + } +} diff --git a/src/pkg/reviewrules/store.go b/src/pkg/reviewrules/store.go index 2ac07fc03e..717201a6f6 100644 --- a/src/pkg/reviewrules/store.go +++ b/src/pkg/reviewrules/store.go @@ -58,6 +58,9 @@ const ( // maxSeenComments bounds the remembered review comments. maxSeenComments = 500 + // maxEvaluatedComments bounds the comment ids rule capture classified. + // Evicting the oldest only means one more, idempotent, classification. + maxEvaluatedComments = 5000 // maxRepliesKept bounds the remembered applied agent replies. maxRepliesKept = 500 // titleRunes bounds a rule's entry title. @@ -159,6 +162,12 @@ type index struct { Comments map[string]string `json:"comments"` Seen map[string]seenComment `json:"seen"` Replies map[string]time.Time `json:"replies"` + // Evaluated are the comments rule capture has classified, keyed like + // Comments, with when. It is separate from the review-feedback sweep's + // handled ids, so a comment the agent already answered is still read for + // rules once, and an index from before this field backfills on its first + // pass (hivecommons/hive#19136). + Evaluated map[string]time.Time `json:"evaluated,omitempty"` } // Config is the capture policy. @@ -275,7 +284,7 @@ func Open(dir, vault string, cfg Config) (*Store, error) { } func newIndex() index { - return index{Next: 1, Comments: map[string]string{}, Seen: map[string]seenComment{}, Replies: map[string]time.Time{}} + return index{Next: 1, Comments: map[string]string{}, Seen: map[string]seenComment{}, Replies: map[string]time.Time{}, Evaluated: map[string]time.Time{}} } func (s *Store) fillMaps() { @@ -291,6 +300,9 @@ func (s *Store) fillMaps() { if s.st.Replies == nil { s.st.Replies = map[string]time.Time{} } + if s.st.Evaluated == nil { + s.st.Evaluated = map[string]time.Time{} + } } // Comment is one review comment offered for capture. @@ -457,6 +469,58 @@ func (s *Store) CaptureComment(c Comment, now time.Time) Result { return s.captureLocked(c, text, ScopeAgent, c.Lane, home, "sweep", confidenceSweep, cl.Signals, now) } +// Evaluated reports whether rule capture already classified this comment. +func (s *Store) Evaluated(repo string, pr int, commentID string) bool { + s.mu.Lock() + defer s.mu.Unlock() + _, ok := s.st.Evaluated[commentKey(repo, pr, commentID)] + return ok +} + +// MarkEvaluated records that rule capture classified these comments, so a +// later pass does not read them again. +func (s *Store) MarkEvaluated(repo string, pr int, commentIDs []string, now time.Time) error { + if len(commentIDs) == 0 { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + return s.txn(func() []*Rule { + for _, id := range commentIDs { + s.st.Evaluated[commentKey(repo, pr, id)] = now + } + if len(s.st.Evaluated) > maxEvaluatedComments { + keys := make([]string, 0, len(s.st.Evaluated)) + for k := range s.st.Evaluated { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + a, b := s.st.Evaluated[keys[i]], s.st.Evaluated[keys[j]] + if !a.Equal(b) { + return a.Before(b) + } + return keys[i] < keys[j] + }) + for _, k := range keys[:len(s.st.Evaluated)-maxEvaluatedComments] { + delete(s.st.Evaluated, k) + } + } + return nil + }) +} + +// RuleForComment returns a copy of the rule this comment was captured as or +// merged into, following merges, or nil when it became none. +func (s *Store) RuleForComment(repo string, pr int, commentID string) *Rule { + s.mu.Lock() + defer s.mu.Unlock() + id, ok := s.st.Comments[commentKey(repo, pr, commentID)] + if !ok { + return nil + } + return s.copyOf(s.resolve(s.byID(id))) +} + // captureLocked creates a rule, or joins an existing near-identical one. func (s *Store) captureLocked(c Comment, text string, sc Scope, appliesTo, home, source string, conf float64, signals []string, now time.Time) Result { trusted := s.trusted(c.Association) diff --git a/src/pkg/reviewrules/store_test.go b/src/pkg/reviewrules/store_test.go index 0d59f7b94d..88aa70328c 100644 --- a/src/pkg/reviewrules/store_test.go +++ b/src/pkg/reviewrules/store_test.go @@ -3,6 +3,7 @@ package reviewrules import ( "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -494,3 +495,53 @@ func TestSimilarity(t *testing.T) { } } } + +// The evaluated ids are the rule index's own record, kept across a reload, and +// a captured comment's rule can be looked up for a later kick +// (hivecommons/hive#19136). +func TestEvaluatedCommentsPersistAndRuleForComment(t *testing.T) { + s, dir, vault := openStore(t, Config{}) + if s.Evaluated("cncf/techdocs", 424, "inline:1001") { + t.Fatal("a fresh index has evaluated nothing: the first pass is a backfill") + } + res := s.CaptureComment(cmt("inline:1001", "OWNER", maintainersComment), t0) + if res.Outcome != OutcomeCaptured { + t.Fatalf("res = %+v", res) + } + if err := s.MarkEvaluated("cncf/techdocs", 424, []string{"inline:1001", "inline:1002"}, t0); err != nil { + t.Fatal(err) + } + re, err := Open(dir, vault, Config{}) + if err != nil { + t.Fatal(err) + } + if !re.Evaluated("cncf/techdocs", 424, "inline:1001") || !re.Evaluated("cncf/techdocs", 424, "inline:1002") { + t.Fatal("evaluated ids lost across reload") + } + if re.Evaluated("cncf/techdocs", 425, "inline:1001") { + t.Fatal("evaluated ids are per PR") + } + if r := re.RuleForComment("cncf/techdocs", 424, "inline:1001"); r == nil || r.ID != res.Rule.ID { + t.Fatalf("rule for comment = %+v, want %s", r, res.Rule.ID) + } + if r := re.RuleForComment("cncf/techdocs", 424, "inline:1002"); r != nil { + t.Fatalf("a comment that became no rule has none: %+v", r) + } +} + +func TestEvaluatedCommentsAreBounded(t *testing.T) { + s, _, _ := openStore(t, Config{}) + ids := make([]string, 0, maxEvaluatedComments+5) + for i := 0; i < maxEvaluatedComments+5; i++ { + ids = append(ids, "inline:"+strconv.Itoa(i)) + } + if err := s.MarkEvaluated("o/r", 1, ids[:5], t0); err != nil { + t.Fatal(err) + } + if err := s.MarkEvaluated("o/r", 1, ids[5:], t0.Add(time.Minute)); err != nil { + t.Fatal(err) + } + if len(s.st.Evaluated) != maxEvaluatedComments || s.Evaluated("o/r", 1, "inline:0") || !s.Evaluated("o/r", 1, ids[len(ids)-1]) { + t.Fatalf("evaluated = %d, want the oldest evicted", len(s.st.Evaluated)) + } +}