Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-19136-review-rule-candidates.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Reviewer-stated rule capture now reads every human comment on the hive's open PRs (inline, review body and plain conversation comments), including ones already handled, in resolved threads or not addressed to the hive, and the rule index tracks which comments it has evaluated so the first pass after upgrade backfills once and later passes classify only new comments ([#19136](https://github.com/hivecommons/hive/issues/19136)).
66 changes: 66 additions & 0 deletions src/cmd/hive/review_feedback_rules_wire_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ func (h *ruleHarness) sweep(t *testing.T) reviewFeedbackSweep {
func (h *ruleHarness) feedback(items ...github.FeedbackItem) {
fb := openDraftFeedback()
fb.Items = items
fb.RuleCandidates = items
h.src.byKey[feedbackTestKey] = fb
}

Expand Down Expand Up @@ -386,3 +387,68 @@ func TestReviewFeedbackSweep_ReplyMustNameCapturedRule(t *testing.T) {
}
})
}

// The maintainers.yaml comment on cncf/techdocs#424 was already named in a
// handled marker, so routing no longer returns it; a plain conversation comment
// does not address the hive. Rule capture reads its own candidate set, so both
// are captured on the first pass (the backfill) without a kick, and a later
// pass does not read them again (hivecommons/hive#19136).
func TestReviewFeedbackSweep_RuleCaptureReadsHandledAndUnaddressedComments(t *testing.T) {
h := newRuleHarness(t, reviewrules.Config{})
plain := ruleItem("conversation:2002", "OWNER", "Going forward, cite the scorecard with a commit-pinned link.")
plain.Kind, plain.ThreadID, plain.Path, plain.Line = github.PRCommentConversation, "", "", 0
fb := openDraftFeedback()
fb.Items = nil
fb.RuleCandidates = []github.FeedbackItem{ruleItem("inline:1001", "MEMBER", maintainersFixture), plain}
h.src.byKey[feedbackTestKey] = fb
res := h.sweep(t).run(context.Background(), parseReviewFeedbackKey)
if res.RulesCaptured != 2 || res.Kicked != 0 || len(h.rules.List()) != 2 {
t.Fatalf("res = %+v rules = %d, want both captured and no kick", res, len(h.rules.List()))
}
if r := h.rules.Get("R-1"); r == nil || r.Text != maintainersRuleText {
t.Fatalf("R-1 = %+v", r)
}
for _, id := range []string{"inline:1001", "conversation:2002"} {
if !h.rules.Evaluated("cncf/techdocs", 424, id) {
t.Fatalf("%s not recorded as evaluated", id)
}
}
if len(h.src.comments) != 1 || !strings.Contains(h.src.comments[0], "R-1") || !strings.Contains(h.src.comments[0], "R-2") {
t.Fatalf("comments = %q, want one acknowledgement naming both rules", h.src.comments)
}

// Later passes classify only new comments.
fb.RuleCandidates = append(fb.RuleCandidates, ruleItem("inline:1003", "OWNER", "The score is too high."))
h.src.byKey[feedbackTestKey] = fb
h.now = h.now.Add(time.Hour)
res = h.sweep(t).run(context.Background(), parseReviewFeedbackKey)
if res.RulesCaptured+res.RulesMerged+res.RulesQueued != 0 || len(h.src.comments) != 1 || !h.rules.Evaluated("cncf/techdocs", 424, "inline:1003") {
t.Fatalf("res = %+v comments = %d", res, len(h.src.comments))
}
}

// A candidate still inside the debounce may be edited; it is read once it is
// quiet, and a kick of a comment captured on an earlier pass still names the rule.
func TestReviewFeedbackSweep_RuleCaptureWaitsForDebounceAndBriefNamesEarlierCapture(t *testing.T) {
h := newRuleHarness(t, reviewrules.Config{})
it := ruleItem("inline:1001", "OWNER", maintainersFixture)
it.CreatedAt = h.now
fb := openDraftFeedback()
fb.Items = nil
fb.RuleCandidates = []github.FeedbackItem{it}
h.src.byKey[feedbackTestKey] = fb
if res := h.sweep(t).run(context.Background(), parseReviewFeedbackKey); res.RulesCaptured != 0 || h.rules.Evaluated("cncf/techdocs", 424, "inline:1001") {
t.Fatalf("a comment still in the debounce was read: %+v", res)
}
h.now = h.now.Add(time.Hour)
if res := h.sweep(t).run(context.Background(), parseReviewFeedbackKey); res.RulesCaptured != 1 {
t.Fatalf("res = %+v", res)
}
fb.Items = []github.FeedbackItem{it}
h.src.byKey[feedbackTestKey] = fb
h.now = h.now.Add(time.Hour)
res := h.sweep(t).run(context.Background(), parseReviewFeedbackKey)
if res.Kicked != 1 || res.RulesCaptured != 0 || !strings.Contains(h.kick.kicks["quality"][0], "R-1 (recorded") {
t.Fatalf("res = %+v kicks = %q", res, h.kick.kicks)
}
}
55 changes: 40 additions & 15 deletions src/cmd/hive/review_feedback_wire.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ func (s reviewFeedbackSweep) run(ctx context.Context, parse func(key string) (re
s.applyRuleReplies(repo, number, fb.HiveReplies)
s.postRuleNotices(ctx, repo, number)
s.checkRuleReply(ctx, key, repo, number, fb.AgentReplies, &res)
s.captureRules(ctx, repo, number, redPRFixOwner(s.Owners[key]), fb.RuleCandidates, &res)
}
if !fb.Open() || len(fb.Items) == 0 {
continue
Expand All @@ -135,10 +136,9 @@ func (s reviewFeedbackSweep) run(ctx context.Context, parse func(key string) (re
s.Logger.Debug("review feedback: burst still arriving, waiting to coalesce", "repo", repo, "pr", number, "comments", len(d.Items))
case reviewfeedback.VerdictCapReached:
res.CapReached++
s.captureRules(ctx, repo, number, redPRFixOwner(s.Owners[key]), d.Items, &res)
s.needsHuman(ctx, key, repo, number, "it has been routed to its agent "+strconv.Itoa(s.Limits.WindowCap)+" times in "+s.Limits.Window.String()+"; further feedback is waiting, and a new comment from a person resets this", d.Items, &res)
case reviewfeedback.VerdictKick:
recorded := s.captureRules(ctx, repo, number, redPRFixOwner(s.Owners[key]), d.Items, &res)
recorded := s.recordedRules(repo, number, d.Items)
if route.Agent == "" {
s.needsHuman(ctx, key, repo, number, route.Reason, d.Items, &res)
continue
Expand Down Expand Up @@ -189,28 +189,38 @@ func (s reviewFeedbackSweep) needsHuman(ctx context.Context, key, repo string, n
res.NeedsHuman++
}

// captureRules is the deterministic half of rule capture (#19046): every
// comment in the burst that states a rule is written to the hive's knowledge
// store whether or not the agent later records it, idempotently per comment.
// It runs for every verdict that carries comments, so a rule is not lost to the
// cap or to an unavailable agent. It returns the rules for the agent's brief.
func (s reviewFeedbackSweep) captureRules(ctx context.Context, repo string, number int, owner string, items []github.FeedbackItem, res *reviewFeedbackResult) []reviewfeedback.RecordedRule {
// captureRules is the deterministic half of rule capture (#19046): every human
// comment on the PR that states a rule is written to the hive's knowledge store
// whether or not the agent later records it, idempotently per comment. It reads
// its own candidate set (#19136), not the routed burst, so a comment that was
// already handled, sits in a resolved thread, or does not address the hive is
// still classified. Each comment is classified once: the rule index keeps the
// evaluated ids, so the first pass over an index without them is a one-time
// backfill and later passes read only new comments. A comment newer than the
// debounce may still be edited and waits for a later pass.
func (s reviewFeedbackSweep) captureRules(ctx context.Context, repo string, number int, owner string, candidates []github.FeedbackItem, res *reviewFeedbackResult) {
if s.Rules == nil {
return nil
return
}
lane := owner
if s.Lane != nil {
lane = s.Lane(owner)
}
managed := s.Managed != nil && s.Managed(repo)
var recorded []reviewfeedback.RecordedRule
for _, it := range items {
var evaluated []string
for _, it := range candidates {
if s.Now.Sub(it.CreatedAt) < s.Limits.Debounce || s.Rules.Evaluated(repo, number, it.ID) {
continue
}
c := reviewrules.Comment{Repo: repo, PR: number, CommentID: it.ID, URL: it.URL, Reviewer: it.Author,
Association: it.AuthorAssociation, Body: it.Body, At: it.CreatedAt, Lane: lane, ManagedRepo: managed}
if err := s.Rules.Remember(c, s.Now); err != nil {
s.Logger.Warn("review feedback: could not remember the comment for rule evidence", "repo", repo, "pr", number, "comment", it.ID, "error", err)
}
r := s.Rules.CaptureComment(c, s.Now)
if r.Outcome != reviewrules.OutcomeRefused {
evaluated = append(evaluated, it.ID) // a refused rule is tried again on a later pass
}
switch r.Outcome {
case reviewrules.OutcomeNotRule:
continue
Expand All @@ -229,13 +239,28 @@ func (s reviewFeedbackSweep) captureRules(ctx context.Context, repo string, numb
"reviewer", it.Author, "association", it.AuthorAssociation, "outcome", string(r.Outcome), "rule", r.Rule.ID, "scope", string(r.Rule.Scope),
"applies_to", r.Rule.AppliesTo, "signals", r.Signals)
}
if r.Rule == nil || (r.Rule.Status != reviewrules.StatusActive && r.Rule.Status != reviewrules.StatusPending) {
}
if err := s.Rules.MarkEvaluated(repo, number, evaluated, s.Now); err != nil {
s.Logger.Warn("review feedback: could not persist the evaluated comment ids; a later pass reads them again", "repo", repo, "pr", number, "error", err)
}
s.postRuleNotices(ctx, repo, number)
}

// recordedRules returns the live rules the routed comments were captured as,
// for the agent's brief, whichever pass captured them.
func (s reviewFeedbackSweep) recordedRules(repo string, number int, items []github.FeedbackItem) []reviewfeedback.RecordedRule {
if s.Rules == nil {
return nil
}
var recorded []reviewfeedback.RecordedRule
for _, it := range items {
r := s.Rules.RuleForComment(repo, number, it.ID)
if r == nil || (r.Status != reviewrules.StatusActive && r.Status != reviewrules.StatusPending) {
continue
}
recorded = append(recorded, reviewfeedback.RecordedRule{ID: r.Rule.ID, Text: r.Rule.Text, Applies: r.Rule.Describe(),
Queued: r.Rule.Status == reviewrules.StatusPending, SourceURL: it.URL})
recorded = append(recorded, reviewfeedback.RecordedRule{ID: r.ID, Text: r.Text, Applies: r.Describe(),
Queued: r.Status == reviewrules.StatusPending, SourceURL: it.URL})
}
s.postRuleNotices(ctx, repo, number)
return recorded
}

Expand Down
10 changes: 7 additions & 3 deletions src/docs/review-feedback-rules.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ leaving it to the agent's discretion ([#19046](https://github.com/hivecommons/hi

## What happens

1. **Classify.** Each comment in a coalesced burst is matched against fixed phrase signals. A comment with at least one is a rule; the rest
1. **Classify.** Each human comment on the PR is matched against fixed phrase signals. A comment with at least one is a rule; the rest
are corrections and are handled as before. The signals are `source-of-truth`, `authoritative`, `always` ("should always", "must always"),
`never` ("should never", "must never"), `going-forward` ("going forward", "from now on", "next time", "for future assessments"),
`capture-request` ("capture this", "document this", "record it"), `assessment-criteria` ("assessment criteria", "assessment knowledge")
Expand All @@ -31,8 +31,12 @@ leaving it to the agent's discretion ([#19046](https://github.com/hivecommons/hi
so the quality agent receives its rules on its next kick; a rule scoped to a workflow reaches the lane or workflow model of that name. The
usual lifecycle filter and `knowledge.agent_scopes` still apply, so drafts, retracted rules and out-of-scope agents never see a rule.

Capture happens when the burst is routed, so it also happens when the per-PR cap was reached or no agent could take the kick. It does not
change the sweep's debounce, cap or thread behavior.
Capture reads its own candidate set, not the routed burst: every human comment on the hive's open PRs in the attribution window (inline,
review body, conversation), whether or not it was already handled, sits in a resolved thread, or addresses the hive. A comment is read once
it has been quiet for the sweep's debounce. The rule index keeps the ids it has evaluated, separately from the sweep's handled ids, so the
first pass after an upgrade is a one-time backfill over the comments still visible on those PRs and later passes read only new comments. So
capture also happens when the per-PR cap was reached, no agent could take the kick, or no kick is due at all. It does not change the sweep's
debounce, cap, routing or thread behavior.

## The agent's structured block

Expand Down
84 changes: 68 additions & 16 deletions src/pkg/github/pr_feedback.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,10 @@ type PRFeedback struct {
// AgentReplies are the hive's comments that carry the handled-ids
// marker: the kicked agent's replies to feedback.
AgentReplies []FeedbackMessage
// RuleCandidates are every human comment on the PR, whatever its handled,
// resolved or addressed state, oldest first. Rule capture reads these
// (hivecommons/hive#19136); kick routing reads Items.
RuleCandidates []FeedbackItem
}

// Open reports whether the PR can still take a revision.
Expand Down Expand Up @@ -172,13 +176,14 @@ func (c *Client) FetchPRFeedback(ctx context.Context, repo string, number int) (
state = "closed"
}
return PRFeedback{
State: state,
Draft: raw.IsDraft,
HeadRef: raw.HeadRefName,
Items: UnansweredFeedback(raw.rawPRConversation, c.isHumanFeedbackAuthor, c.isAgentCommentAuthor, c.addressesHive),
Source: source,
HiveReplies: HiveRuleReplies(raw.rawPRConversation, c.isHiveAuthor),
AgentReplies: HiveFeedbackReplies(raw.rawPRConversation, c.isHiveAuthor),
State: state,
Draft: raw.IsDraft,
HeadRef: raw.HeadRefName,
Items: UnansweredFeedback(raw.rawPRConversation, c.isHumanFeedbackAuthor, c.isAgentCommentAuthor, c.addressesHive),
Source: source,
HiveReplies: HiveRuleReplies(raw.rawPRConversation, c.isHiveAuthor),
AgentReplies: HiveFeedbackReplies(raw.rawPRConversation, c.isHiveAuthor),
RuleCandidates: RuleCandidates(raw.rawPRConversation, c.isHumanFeedbackAuthor),
}, nil
}

Expand Down Expand Up @@ -252,15 +257,7 @@ func UnansweredFeedback(raw rawPRConversation, isHuman func(*rawPRCommentAuthor)
}
}
}
human := func(rc rawPRComment) bool {
if rc.Author == nil || strings.TrimSpace(rc.Body) == "" || HasAttributionTrailer(rc.Body) {
return false
}
if !feedbackTrustedAssociations[strings.ToUpper(rc.AuthorAssociation)] {
return false
}
return isHuman != nil && isHuman(rc.Author)
}
human := func(rc rawPRComment) bool { return humanFeedback(rc, isHuman) }
var out []FeedbackItem
add := func(kind PRCommentKind, rc rawPRComment, thread, path string, line int, ctxThread []FeedbackMessage) {
id := string(kind) + ":" + strconv.FormatInt(rc.DatabaseID, 10)
Expand Down Expand Up @@ -321,6 +318,61 @@ func UnansweredFeedback(raw rawPRConversation, isHuman func(*rawPRCommentAuthor)
return out
}

// humanFeedback reports whether a comment is a person's: non-empty, from a
// human (isHuman) with a trusted association, and without a hive attribution
// trailer.
func humanFeedback(rc rawPRComment, isHuman func(*rawPRCommentAuthor) bool) bool {
if rc.Author == nil || strings.TrimSpace(rc.Body) == "" || HasAttributionTrailer(rc.Body) {
return false
}
if !feedbackTrustedAssociations[strings.ToUpper(rc.AuthorAssociation)] {
return false
}
return isHuman != nil && isHuman(rc.Author)
}

// RuleCandidates returns every human comment on a PR for rule capture
// (hivecommons/hive#19136): inline comments, non-empty review bodies of any
// state, and conversation comments, with the same author checks as
// UnansweredFeedback but none of its routing gates. A comment named in a
// handled marker, older than the hive's reply in its thread, in a resolved
// thread, or a conversation comment not addressed to the hive is still a
// candidate, because a reviewer can state a rule in any of them. Which
// candidates were already classified is the rule index's concern.
func RuleCandidates(raw rawPRConversation, isHuman func(*rawPRCommentAuthor) bool) []FeedbackItem {
var out []FeedbackItem
add := func(kind PRCommentKind, rc rawPRComment, thread, path string, line int) {
if !humanFeedback(rc, isHuman) {
return
}
out = append(out, FeedbackItem{ID: string(kind) + ":" + strconv.FormatInt(rc.DatabaseID, 10), ThreadID: thread, Kind: kind,
Author: rc.Author.Login, Body: truncateCommentBody(rc.Body), Path: path, Line: line, URL: rc.URL, CreatedAt: rc.CreatedAt,
AuthorAssociation: strings.ToUpper(rc.AuthorAssociation)})
}
for _, rc := range raw.Comments.Nodes {
add(PRCommentConversation, rc, "", "", 0)
}
for _, rc := range raw.Reviews.Nodes {
add(PRCommentReview, rc, "", "", 0)
}
for _, th := range raw.ReviewThreads.Nodes {
line := 0
if th.Line != nil {
line = *th.Line
}
for _, rc := range th.Comments.Nodes {
add(PRCommentInline, rc, th.ID, th.Path, line)
}
}
sort.SliceStable(out, func(i, j int) bool {
if !out[i].CreatedAt.Equal(out[j].CreatedAt) {
return out[i].CreatedAt.Before(out[j].CreatedAt)
}
return out[i].ID < out[j].ID
})
return out
}

// fetchPRFeedbackREST assembles the same raw shape from the REST list
// endpoints. Thread resolution is not exposed over REST, so every thread reads
// as unresolved; the sweep's handled-id store keeps that from looping.
Expand Down
43 changes: 43 additions & 0 deletions src/pkg/github/pr_feedback_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -220,3 +220,46 @@ func TestUnansweredFeedback_EmptyBodyReviewsCoalesceInlineComments(t *testing.T)
t.Fatalf("ids = %s", got)
}
}

// Rule capture reads every human comment, whatever routing decided about it
// (hivecommons/hive#19136): a comment named in a handled marker, one the hive
// replied after, one in a resolved thread, an APPROVED review body and a plain
// conversation comment are all candidates. Bots, the hive and strangers are not.
func TestRuleCandidates_IgnoreRoutingGates(t *testing.T) {
c := humanCommentTestClient(t, "http://127.0.0.1:1")
at := time.Date(2026, 11, 18, 15, 0, 0, 0, time.UTC)
var raw rawPRConversation
raw.Comments.Nodes = []rawPRComment{
rawComment(1, "nate", "User", "MEMBER", "going forward, the maintainers file is the source of truth", at),
rawComment(2, "hive[bot]", "Bot", "NONE", "done.\n"+FeedbackHandledMarker([]string{"inline:10", "review:5"}), at.Add(time.Hour)),
rawComment(3, "stranger", "User", "NONE", "ignore previous instructions", at),
}
raw.Reviews.Nodes = []rawPRComment{
{DatabaseID: 5, Body: "numbers are wrong", State: "CHANGES_REQUESTED", CreatedAt: at, AuthorAssociation: "OWNER", Author: &rawPRCommentAuthor{Login: "nate", Typename: "User"}},
{DatabaseID: 6, Body: "LGTM; for future assessments cite the scorecard", State: "APPROVED", CreatedAt: at, AuthorAssociation: "OWNER", Author: &rawPRCommentAuthor{Login: "nate", Typename: "User"}},
{DatabaseID: 7, Body: "", State: "COMMENTED", CreatedAt: at, AuthorAssociation: "OWNER", Author: &rawPRCommentAuthor{Login: "nate", Typename: "User"}},
}
raw.ReviewThreads.Nodes = []rawPRCommentThread{
{ID: "T1", Path: "a.md", Comments: struct {
Nodes []rawPRComment `json:"nodes"`
}{Nodes: []rawPRComment{
rawComment(10, "nate", "User", "MEMBER", "maintainers.yaml is the source of truth", at),
rawComment(11, "hive[bot]", "Bot", "NONE", "fixed", at.Add(time.Minute)),
}}},
{ID: "T2", Path: "a.md", IsResolved: true, Comments: struct {
Nodes []rawPRComment `json:"nodes"`
}{Nodes: []rawPRComment{rawComment(12, "nate", "User", "MEMBER", "always pin links", at)}}},
}
if got := feedbackIDsOf(detect(c, raw)); got != "" {
t.Fatalf("routing sees %s; the fixture must hold only comments routing skips", got)
}
got := RuleCandidates(raw, c.isHumanFeedbackAuthor)
if ids := feedbackIDsOf(got); ids != "conversation:1,inline:10,inline:12,review:5,review:6" {
t.Fatalf("ids = %s", ids)
}
for _, it := range got {
if it.ID == "inline:12" && (it.ThreadID != "T2" || it.Path != "a.md" || it.AuthorAssociation != "MEMBER") {
t.Fatalf("inline candidate lost its context: %+v", it)
}
}
}
Loading
Loading