diff --git a/changelog.d/fixed-19048-review-rule-reply-check.md b/changelog.d/fixed-19048-review-rule-reply-check.md new file mode 100644 index 0000000000..ee65eb1e8e --- /dev/null +++ b/changelog.d/fixed-19048-review-rule-reply-check.md @@ -0,0 +1 @@ +- After a review-feedback kick that carried captured reviewer rules, the sweep now checks the agent's reply names each rule id and, when it does not (or no reply arrives within 6 hours), logs it and posts one needs-human note on the PR ([#19048](https://github.com/hivecommons/hive/issues/19048)). diff --git a/src/cmd/hive/review_feedback_rules_wire_test.go b/src/cmd/hive/review_feedback_rules_wire_test.go index 545da0c7e2..532586eda0 100644 --- a/src/cmd/hive/review_feedback_rules_wire_test.go +++ b/src/cmd/hive/review_feedback_rules_wire_test.go @@ -352,3 +352,37 @@ func TestRecordedRulesBriefShape(t *testing.T) { t.Fatalf("brief:\n%s", brief) } } + +// After the kick, the agent's reply must name the captured rule. A reply that +// does not raises one needs-human note; one that does raises nothing. +func TestReviewFeedbackSweep_ReplyMustNameCapturedRule(t *testing.T) { + run := func(t *testing.T, replyBody string) (reviewFeedbackResult, []string) { + h := newRuleHarness(t, reviewrules.Config{}) + h.feedback(ruleItem("inline:1001", "OWNER", maintainersFixture)) + s := h.sweep(t) + s.run(context.Background(), parseReviewFeedbackKey) + before := len(h.src.comments) + fb := openDraftFeedback() + fb.Items = nil + fb.AgentReplies = []github.FeedbackMessage{{ID: "conversation:9", Hive: true, CreatedAt: h.now.Add(time.Minute), + Body: replyBody + "\n" + github.FeedbackHandledMarker([]string{"inline:1001"})}} + h.src.byKey[feedbackTestKey] = fb + s.Now = h.now.Add(time.Hour) + var res reviewFeedbackResult + for i := 0; i < 2; i++ { + res = s.run(context.Background(), parseReviewFeedbackKey) + } + return res, h.src.comments[before:] + } + t.Run("names it", func(t *testing.T) { + if _, notes := run(t, "Recorded as R-1, applied here."); len(notes) != 0 { + t.Fatalf("notes = %q, want none", notes) + } + }) + t.Run("does not name it", func(t *testing.T) { + _, notes := run(t, "Fixed the file.") + if len(notes) != 1 || !strings.Contains(notes[0], github.FeedbackNeedsHumanMarker) || !strings.Contains(notes[0], "R-1") { + t.Fatalf("notes = %q, want exactly one needs-human note naming R-1", notes) + } + }) +} diff --git a/src/cmd/hive/review_feedback_wire.go b/src/cmd/hive/review_feedback_wire.go index 34ef56b2df..d6e804a49c 100644 --- a/src/cmd/hive/review_feedback_wire.go +++ b/src/cmd/hive/review_feedback_wire.go @@ -82,6 +82,7 @@ type reviewFeedbackSweep struct { type reviewFeedbackResult struct { Checked, Kicked, Debounced, CapReached, NeedsHuman, Errors int RulesCaptured, RulesQueued, RulesMerged int + ReplyMissingRules int NextCursor int } @@ -122,6 +123,7 @@ func (s reviewFeedbackSweep) run(ctx context.Context, parse func(key string) (re if s.Rules != nil && fb.Open() { s.applyRuleReplies(repo, number, fb.HiveReplies) s.postRuleNotices(ctx, repo, number) + s.checkRuleReply(ctx, key, repo, number, fb.AgentReplies, &res) } if !fb.Open() || len(fb.Items) == 0 { continue @@ -153,6 +155,15 @@ func (s reviewFeedbackSweep) run(ctx context.Context, parse func(key string) (re if err := s.Store.RecordKick(key, d.Items, s.Now); err != nil { s.Logger.Warn("review feedback: could not persist handled ids; a later pass may re-kick", "repo", repo, "pr", number, "error", err) } + if len(recorded) > 0 { + ids := make([]string, 0, len(recorded)) + for _, r := range recorded { + ids = append(ids, r.ID) + } + if err := s.Store.RecordPendingReply(key, ids, s.Now); err != nil { + s.Logger.Warn("review feedback: could not persist the pending reply check", "repo", repo, "pr", number, "error", err) + } + } res.Kicked++ for _, it := range d.Items { s.Logger.Info("review feedback: agent kicked to respond", "repo", repo, "pr", number, "agent", route.Agent, @@ -228,6 +239,31 @@ func (s reviewFeedbackSweep) captureRules(ctx context.Context, repo string, numb return recorded } +// checkRuleReply verifies the agent's reply after a kick that carried captured +// rules names each of them. When it does not, the miss is logged and one +// needs-human note is posted; the check is then cleared. +func (s reviewFeedbackSweep) checkRuleReply(ctx context.Context, key, repo string, number int, replies []github.FeedbackMessage, res *reviewFeedbackResult) { + p, ok := s.Store.PendingReplyFor(key) + if !ok { + return + } + verdict, missing := reviewfeedback.CheckReply(p, replies, s.Now) + if verdict == reviewfeedback.ReplyWaiting { + return + } + if verdict == reviewfeedback.ReplyMissing { + s.Logger.Warn("review feedback: agent reply does not name the captured rules, asking a person", "repo", repo, "pr", number, "rules", missing) + if err := s.Source.CreateIssueComment(ctx, repo, number, reviewfeedback.ReplyCheckNote(missing)); err != nil { // writeback:solicited a person's review feedback on this hive-authored PR + s.Logger.Warn("review feedback: reply-check note not posted, retrying next pass", "repo", repo, "pr", number, "error", err) + return + } + res.ReplyMissingRules++ + } + if err := s.Store.ClearPendingReply(key, s.Now); err != nil { + s.Logger.Warn("review feedback: could not clear the pending reply check", "repo", repo, "pr", number, "error", err) + } +} + // applyRuleReplies reads the structured block in the agent's own replies on the // PR. Each reply is applied once; an entry that does not validate, or does not // cite a comment the hive saw, is logged and dropped. diff --git a/src/docs/notifications.md b/src/docs/notifications.md index 9bd214b2d6..c028226c8b 100644 --- a/src/docs/notifications.md +++ b/src/docs/notifications.md @@ -538,6 +538,7 @@ without a class. Review submissions and review requests are always counted as | `main.maintainHeldPRMergeability` | `attention` | | | `main.rfcCoverageForge.CreateComment` | `attention` | | | `main.runEscalationSweep` | `attention` | | +| `main.reviewFeedbackSweep.checkRuleReply` | `attention` | | | `main.reviewFeedbackSweep.needsHuman` | `attention` | | | `main.reviewFeedbackSweep.proposeRuleHome` | `attention` | | | `prfollowup.postStaleHeadPushNotes` | `attention` | | diff --git a/src/pkg/github/notification_write_sites.go b/src/pkg/github/notification_write_sites.go index 3f69845a7e..69fe84fc0d 100644 --- a/src/pkg/github/notification_write_sites.go +++ b/src/pkg/github/notification_write_sites.go @@ -101,6 +101,7 @@ var notificationWriteSites = map[string]NotificationWriteSite{ "main.maintainHeldPRMergeability": {Class: NotificationClassAttention}, "main.rfcCoverageForge.CreateComment": {Class: NotificationClassAttention}, "main.runEscalationSweep": {Class: NotificationClassAttention}, + "main.reviewFeedbackSweep.checkRuleReply": {Class: NotificationClassAttention}, "main.reviewFeedbackSweep.needsHuman": {Class: NotificationClassAttention}, "main.reviewFeedbackSweep.proposeRuleHome": {Class: NotificationClassAttention}, "prfollowup.postStaleHeadPushNotes": {Class: NotificationClassAttention}, diff --git a/src/pkg/github/pr_feedback.go b/src/pkg/github/pr_feedback.go index 3555d4de56..664f1df2d6 100644 --- a/src/pkg/github/pr_feedback.go +++ b/src/pkg/github/pr_feedback.go @@ -100,6 +100,9 @@ type PRFeedback struct { // block (the structured statement of the rules a reviewer stated). They // are read even when no human comment is unanswered. HiveReplies []FeedbackMessage + // AgentReplies are the hive's comments that carry the handled-ids + // marker: the kicked agent's replies to feedback. + AgentReplies []FeedbackMessage } // Open reports whether the PR can still take a revision. @@ -169,17 +172,20 @@ func (c *Client) FetchPRFeedback(ctx context.Context, repo string, number int) ( state = "closed" } return PRFeedback{ - State: state, - Draft: raw.IsDraft, - HeadRef: raw.HeadRefName, - Items: UnansweredFeedback(raw.rawPRConversation, c.isHumanFeedbackAuthor, c.isAgentCommentAuthor, c.addressesHive), - Source: source, - HiveReplies: HiveRuleReplies(raw.rawPRConversation, func(a *rawPRCommentAuthor) bool { - return a != nil && c.isHiveLogin(strings.TrimSpace(a.Login)) - }), + State: state, + Draft: raw.IsDraft, + HeadRef: raw.HeadRefName, + Items: UnansweredFeedback(raw.rawPRConversation, c.isHumanFeedbackAuthor, c.isAgentCommentAuthor, c.addressesHive), + Source: source, + HiveReplies: HiveRuleReplies(raw.rawPRConversation, c.isHiveAuthor), + AgentReplies: HiveFeedbackReplies(raw.rawPRConversation, c.isHiveAuthor), }, nil } +func (c *Client) isHiveAuthor(a *rawPRCommentAuthor) bool { + return a != nil && c.isHiveLogin(strings.TrimSpace(a.Login)) +} + // addressesHive reports whether a conversation comment is aimed at the hive: // it mentions the App bot or the AI author account, or starts with /hive. // Inline review comments and review bodies on a hive PR are always aimed at diff --git a/src/pkg/github/pr_feedback_replies.go b/src/pkg/github/pr_feedback_replies.go index fa7a06c160..2a3820e8df 100644 --- a/src/pkg/github/pr_feedback_replies.go +++ b/src/pkg/github/pr_feedback_replies.go @@ -16,9 +16,20 @@ const FeedbackRulesMarker = "hive:review-rules" // only the hive's own logins: the block is read as the agent's statement, so a // comment merely quoting the marker from a person is not one. func HiveRuleReplies(raw rawPRConversation, isHive func(*rawPRCommentAuthor) bool) []FeedbackMessage { + return hiveComments(raw, isHive, FeedbackRulesMarker) +} + +// HiveFeedbackReplies returns the hive's own comments that carry the +// handled-ids marker the kicked agent ends every reply with, so the sweep can +// tell the agent's reply from the hive's own notices (hivecommons/hive#19048). +func HiveFeedbackReplies(raw rawPRConversation, isHive func(*rawPRCommentAuthor) bool) []FeedbackMessage { + return hiveComments(raw, isHive, FeedbackHandledMarkerPrefix) +} + +func hiveComments(raw rawPRConversation, isHive func(*rawPRCommentAuthor) bool, marker string) []FeedbackMessage { var out []FeedbackMessage add := func(kind PRCommentKind, rc rawPRComment) { - if rc.Author == nil || !isHive(rc.Author) || !strings.Contains(rc.Body, FeedbackRulesMarker) { + if rc.Author == nil || !isHive(rc.Author) || !strings.Contains(rc.Body, marker) { return } out = append(out, FeedbackMessage{ID: string(kind) + ":" + strconv.FormatInt(rc.DatabaseID, 10), Author: rc.Author.Login, diff --git a/src/pkg/reviewfeedback/reviewfeedback.go b/src/pkg/reviewfeedback/reviewfeedback.go index 1cbbf96c6f..ffc12081a8 100644 --- a/src/pkg/reviewfeedback/reviewfeedback.go +++ b/src/pkg/reviewfeedback/reviewfeedback.go @@ -18,6 +18,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "sort" "strconv" "strings" @@ -135,6 +136,112 @@ type PRState struct { CapHitAt time.Time `json:"cap_hit_at,omitempty"` // NeedsHumanNoted is set once the needs-human note was posted. NeedsHumanNoted bool `json:"needs_human_noted,omitempty"` + // Pending is the reply check owed for the last kick that carried + // captured rules (hivecommons/hive#19048). + Pending *PendingReply `json:"pending,omitempty"` +} + +// PendingReply is the rules a kick promised to be named in the agent's reply. +type PendingReply struct { + RuleIDs []string `json:"rule_ids"` + KickedAt time.Time `json:"kicked_at"` +} + +// ReplyGrace is how long the agent has to reply after a kick before a missing +// reply counts as a failure to name the rules. +const ReplyGrace = 6 * time.Hour + +// Reply check verdicts. +const ( + ReplyWaiting = "waiting" + ReplyOK = "ok" + ReplyMissing = "missing" +) + +// CheckReply judges the agent's replies against a pending check. Only replies +// posted after the kick count. Every rule id must be named in them; with no +// reply yet the check waits until ReplyGrace has passed. Missing returns the +// ids not named. +func CheckReply(p PendingReply, replies []github.FeedbackMessage, now time.Time) (string, []string) { + var text strings.Builder + replied := false + for _, r := range replies { + if r.CreatedAt.After(p.KickedAt) { + replied = true + text.WriteString(r.Body + "\n") + } + } + if !replied { + if now.Sub(p.KickedAt) < ReplyGrace { + return ReplyWaiting, nil + } + return ReplyMissing, p.RuleIDs + } + var missing []string + for _, id := range p.RuleIDs { + if !regexp.MustCompile(`(?i)\b` + regexp.QuoteMeta(id) + `\b`).MatchString(text.String()) { + missing = append(missing, id) + } + } + if len(missing) > 0 { + return ReplyMissing, missing + } + return ReplyOK, nil +} + +// RecordPendingReply notes that the kick at now carried the rule ids, so the +// reply is checked on later passes. Ids of an unanswered earlier kick stay. +func (s *Store) RecordPendingReply(key string, ids []string, now time.Time) error { + if len(ids) == 0 { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + st := s.get(key) + merged := append([]string(nil), ids...) + if st.Pending != nil { + for _, id := range st.Pending.RuleIDs { + dup := false + for _, m := range merged { + dup = dup || m == id + } + if !dup { + merged = append(merged, id) + } + } + } + sort.Strings(merged) + st.Pending = &PendingReply{RuleIDs: merged, KickedAt: now} + return s.saveLocked(now) +} + +// PendingReplyFor returns the pending reply check for key, if any. +func (s *Store) PendingReplyFor(key string) (PendingReply, bool) { + s.mu.Lock() + defer s.mu.Unlock() + if st := s.PRs[key]; st != nil && st.Pending != nil { + return *st.Pending, true + } + return PendingReply{}, false +} + +// ClearPendingReply drops the pending check once it was resolved or raised. +func (s *Store) ClearPendingReply(key string, now time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + st := s.PRs[key] + if st == nil || st.Pending == nil { + return nil + } + st.Pending = nil + return s.saveLocked(now) +} + +// ReplyCheckNote is the needs-human note posted when the agent's reply did not +// name the rules the hive captured. +func ReplyCheckNote(missing []string) string { + return github.FeedbackNeedsHumanMarker + "\nThe hive recorded reviewer-stated rules (" + strings.Join(missing, ", ") + + ") in its knowledge, but the agent's reply on this PR does not name them. A person needs to check that the reviewer's rule was acknowledged.\n" } // Store persists PRState per "owner/repo#number". diff --git a/src/pkg/reviewfeedback/reviewfeedback_test.go b/src/pkg/reviewfeedback/reviewfeedback_test.go index 5d278814d1..94666f19da 100644 --- a/src/pkg/reviewfeedback/reviewfeedback_test.go +++ b/src/pkg/reviewfeedback/reviewfeedback_test.go @@ -307,3 +307,52 @@ func TestBuildBrief_TruncatesLongBodiesAndShowsThreadlessInlineItem(t *testing.T t.Fatal("thread header must carry path:line and message url") } } + +func TestCheckReply(t *testing.T) { + kicked := time.Now().Add(-time.Hour) + p := PendingReply{RuleIDs: []string{"R-3"}, KickedAt: kicked} + reply := func(body string, at time.Time) []github.FeedbackMessage { + return []github.FeedbackMessage{{ID: "conversation:1", Body: body, CreatedAt: at}} + } + cases := []struct { + name string + replies []github.FeedbackMessage + now time.Time + want string + }{ + {"names the rule", reply("Captured as R-3.", kicked.Add(time.Minute)), time.Now(), ReplyOK}, + {"names another rule", reply("Captured as R-30.", kicked.Add(time.Minute)), time.Now(), ReplyMissing}, + {"does not name it", reply("Fixed.", kicked.Add(time.Minute)), time.Now(), ReplyMissing}, + {"older reply does not count", reply("R-3", kicked.Add(-time.Minute)), time.Now(), ReplyWaiting}, + {"no reply within grace", nil, time.Now(), ReplyWaiting}, + {"no reply after grace", nil, kicked.Add(ReplyGrace + time.Minute), ReplyMissing}, + } + for _, tc := range cases { + if got, _ := CheckReply(p, tc.replies, tc.now); got != tc.want { + t.Errorf("%s: verdict = %s, want %s", tc.name, got, tc.want) + } + } +} + +func TestStorePendingReplyPersistsAndMerges(t *testing.T) { + dir := t.TempDir() + s, _ := Open(dir) + now := time.Now() + if err := s.RecordPendingReply("o/r#1", []string{"R-2"}, now); err != nil { + t.Fatal(err) + } + if err := s.RecordPendingReply("o/r#1", []string{"R-1"}, now.Add(time.Minute)); err != nil { + t.Fatal(err) + } + s2, _ := Open(dir) + p, ok := s2.PendingReplyFor("o/r#1") + if !ok || strings.Join(p.RuleIDs, ",") != "R-1,R-2" { + t.Fatalf("pending = %+v ok=%v", p, ok) + } + if err := s2.ClearPendingReply("o/r#1", now); err != nil { + t.Fatal(err) + } + if _, ok := s2.PendingReplyFor("o/r#1"); ok { + t.Fatal("pending not cleared") + } +}