Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-19048-review-rule-reply-check.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- After a review-feedback kick that carried captured reviewer rules, the sweep now checks the agent's reply names each rule id and, when it does not (or no reply arrives within 6 hours), logs it and posts one needs-human note on the PR ([#19048](https://github.com/hivecommons/hive/issues/19048)).
34 changes: 34 additions & 0 deletions src/cmd/hive/review_feedback_rules_wire_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -352,3 +352,37 @@ func TestRecordedRulesBriefShape(t *testing.T) {
t.Fatalf("brief:\n%s", brief)
}
}

// After the kick, the agent's reply must name the captured rule. A reply that
// does not raises one needs-human note; one that does raises nothing.
func TestReviewFeedbackSweep_ReplyMustNameCapturedRule(t *testing.T) {
run := func(t *testing.T, replyBody string) (reviewFeedbackResult, []string) {
h := newRuleHarness(t, reviewrules.Config{})
h.feedback(ruleItem("inline:1001", "OWNER", maintainersFixture))
s := h.sweep(t)
s.run(context.Background(), parseReviewFeedbackKey)
before := len(h.src.comments)
fb := openDraftFeedback()
fb.Items = nil
fb.AgentReplies = []github.FeedbackMessage{{ID: "conversation:9", Hive: true, CreatedAt: h.now.Add(time.Minute),
Body: replyBody + "\n" + github.FeedbackHandledMarker([]string{"inline:1001"})}}
h.src.byKey[feedbackTestKey] = fb
s.Now = h.now.Add(time.Hour)
var res reviewFeedbackResult
for i := 0; i < 2; i++ {
res = s.run(context.Background(), parseReviewFeedbackKey)
}
return res, h.src.comments[before:]
}
t.Run("names it", func(t *testing.T) {
if _, notes := run(t, "Recorded as R-1, applied here."); len(notes) != 0 {
t.Fatalf("notes = %q, want none", notes)
}
})
t.Run("does not name it", func(t *testing.T) {
_, notes := run(t, "Fixed the file.")
if len(notes) != 1 || !strings.Contains(notes[0], github.FeedbackNeedsHumanMarker) || !strings.Contains(notes[0], "R-1") {
t.Fatalf("notes = %q, want exactly one needs-human note naming R-1", notes)
}
})
}
36 changes: 36 additions & 0 deletions src/cmd/hive/review_feedback_wire.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ type reviewFeedbackSweep struct {
type reviewFeedbackResult struct {
Checked, Kicked, Debounced, CapReached, NeedsHuman, Errors int
RulesCaptured, RulesQueued, RulesMerged int
ReplyMissingRules int
NextCursor int
}

Expand Down Expand Up @@ -122,6 +123,7 @@ func (s reviewFeedbackSweep) run(ctx context.Context, parse func(key string) (re
if s.Rules != nil && fb.Open() {
s.applyRuleReplies(repo, number, fb.HiveReplies)
s.postRuleNotices(ctx, repo, number)
s.checkRuleReply(ctx, key, repo, number, fb.AgentReplies, &res)
}
if !fb.Open() || len(fb.Items) == 0 {
continue
Expand Down Expand Up @@ -153,6 +155,15 @@ func (s reviewFeedbackSweep) run(ctx context.Context, parse func(key string) (re
if err := s.Store.RecordKick(key, d.Items, s.Now); err != nil {
s.Logger.Warn("review feedback: could not persist handled ids; a later pass may re-kick", "repo", repo, "pr", number, "error", err)
}
if len(recorded) > 0 {
ids := make([]string, 0, len(recorded))
for _, r := range recorded {
ids = append(ids, r.ID)
}
if err := s.Store.RecordPendingReply(key, ids, s.Now); err != nil {
s.Logger.Warn("review feedback: could not persist the pending reply check", "repo", repo, "pr", number, "error", err)
}
}
res.Kicked++
for _, it := range d.Items {
s.Logger.Info("review feedback: agent kicked to respond", "repo", repo, "pr", number, "agent", route.Agent,
Expand Down Expand Up @@ -228,6 +239,31 @@ func (s reviewFeedbackSweep) captureRules(ctx context.Context, repo string, numb
return recorded
}

// checkRuleReply verifies the agent's reply after a kick that carried captured
// rules names each of them. When it does not, the miss is logged and one
// needs-human note is posted; the check is then cleared.
func (s reviewFeedbackSweep) checkRuleReply(ctx context.Context, key, repo string, number int, replies []github.FeedbackMessage, res *reviewFeedbackResult) {
p, ok := s.Store.PendingReplyFor(key)
if !ok {
return
}
verdict, missing := reviewfeedback.CheckReply(p, replies, s.Now)
if verdict == reviewfeedback.ReplyWaiting {
return
}
if verdict == reviewfeedback.ReplyMissing {
s.Logger.Warn("review feedback: agent reply does not name the captured rules, asking a person", "repo", repo, "pr", number, "rules", missing)
if err := s.Source.CreateIssueComment(ctx, repo, number, reviewfeedback.ReplyCheckNote(missing)); err != nil { // writeback:solicited a person's review feedback on this hive-authored PR
s.Logger.Warn("review feedback: reply-check note not posted, retrying next pass", "repo", repo, "pr", number, "error", err)
return
}
res.ReplyMissingRules++
}
if err := s.Store.ClearPendingReply(key, s.Now); err != nil {
s.Logger.Warn("review feedback: could not clear the pending reply check", "repo", repo, "pr", number, "error", err)
}
}

// applyRuleReplies reads the structured block in the agent's own replies on the
// PR. Each reply is applied once; an entry that does not validate, or does not
// cite a comment the hive saw, is logged and dropped.
Expand Down
1 change: 1 addition & 0 deletions src/docs/notifications.md
Original file line number Diff line number Diff line change
Expand Up @@ -538,6 +538,7 @@ without a class. Review submissions and review requests are always counted as
| `main.maintainHeldPRMergeability` | `attention` | |
| `main.rfcCoverageForge.CreateComment` | `attention` | |
| `main.runEscalationSweep` | `attention` | |
| `main.reviewFeedbackSweep.checkRuleReply` | `attention` | |
| `main.reviewFeedbackSweep.needsHuman` | `attention` | |
| `main.reviewFeedbackSweep.proposeRuleHome` | `attention` | |
| `prfollowup.postStaleHeadPushNotes` | `attention` | |
Expand Down
1 change: 1 addition & 0 deletions src/pkg/github/notification_write_sites.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ var notificationWriteSites = map[string]NotificationWriteSite{
"main.maintainHeldPRMergeability": {Class: NotificationClassAttention},
"main.rfcCoverageForge.CreateComment": {Class: NotificationClassAttention},
"main.runEscalationSweep": {Class: NotificationClassAttention},
"main.reviewFeedbackSweep.checkRuleReply": {Class: NotificationClassAttention},
"main.reviewFeedbackSweep.needsHuman": {Class: NotificationClassAttention},
"main.reviewFeedbackSweep.proposeRuleHome": {Class: NotificationClassAttention},
"prfollowup.postStaleHeadPushNotes": {Class: NotificationClassAttention},
Expand Down
22 changes: 14 additions & 8 deletions src/pkg/github/pr_feedback.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,9 @@ type PRFeedback struct {
// block (the structured statement of the rules a reviewer stated). They
// are read even when no human comment is unanswered.
HiveReplies []FeedbackMessage
// AgentReplies are the hive's comments that carry the handled-ids
// marker: the kicked agent's replies to feedback.
AgentReplies []FeedbackMessage
}

// Open reports whether the PR can still take a revision.
Expand Down Expand Up @@ -169,17 +172,20 @@ func (c *Client) FetchPRFeedback(ctx context.Context, repo string, number int) (
state = "closed"
}
return PRFeedback{
State: state,
Draft: raw.IsDraft,
HeadRef: raw.HeadRefName,
Items: UnansweredFeedback(raw.rawPRConversation, c.isHumanFeedbackAuthor, c.isAgentCommentAuthor, c.addressesHive),
Source: source,
HiveReplies: HiveRuleReplies(raw.rawPRConversation, func(a *rawPRCommentAuthor) bool {
return a != nil && c.isHiveLogin(strings.TrimSpace(a.Login))
}),
State: state,
Draft: raw.IsDraft,
HeadRef: raw.HeadRefName,
Items: UnansweredFeedback(raw.rawPRConversation, c.isHumanFeedbackAuthor, c.isAgentCommentAuthor, c.addressesHive),
Source: source,
HiveReplies: HiveRuleReplies(raw.rawPRConversation, c.isHiveAuthor),
AgentReplies: HiveFeedbackReplies(raw.rawPRConversation, c.isHiveAuthor),
}, nil
}

func (c *Client) isHiveAuthor(a *rawPRCommentAuthor) bool {
return a != nil && c.isHiveLogin(strings.TrimSpace(a.Login))
}

// addressesHive reports whether a conversation comment is aimed at the hive:
// it mentions the App bot or the AI author account, or starts with /hive.
// Inline review comments and review bodies on a hive PR are always aimed at
Expand Down
13 changes: 12 additions & 1 deletion src/pkg/github/pr_feedback_replies.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,20 @@ const FeedbackRulesMarker = "hive:review-rules"
// only the hive's own logins: the block is read as the agent's statement, so a
// comment merely quoting the marker from a person is not one.
func HiveRuleReplies(raw rawPRConversation, isHive func(*rawPRCommentAuthor) bool) []FeedbackMessage {
return hiveComments(raw, isHive, FeedbackRulesMarker)
}

// HiveFeedbackReplies returns the hive's own comments that carry the
// handled-ids marker the kicked agent ends every reply with, so the sweep can
// tell the agent's reply from the hive's own notices (hivecommons/hive#19048).
func HiveFeedbackReplies(raw rawPRConversation, isHive func(*rawPRCommentAuthor) bool) []FeedbackMessage {
return hiveComments(raw, isHive, FeedbackHandledMarkerPrefix)
}

func hiveComments(raw rawPRConversation, isHive func(*rawPRCommentAuthor) bool, marker string) []FeedbackMessage {
var out []FeedbackMessage
add := func(kind PRCommentKind, rc rawPRComment) {
if rc.Author == nil || !isHive(rc.Author) || !strings.Contains(rc.Body, FeedbackRulesMarker) {
if rc.Author == nil || !isHive(rc.Author) || !strings.Contains(rc.Body, marker) {
return
}
out = append(out, FeedbackMessage{ID: string(kind) + ":" + strconv.FormatInt(rc.DatabaseID, 10), Author: rc.Author.Login,
Expand Down
107 changes: 107 additions & 0 deletions src/pkg/reviewfeedback/reviewfeedback.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
Expand Down Expand Up @@ -135,6 +136,112 @@ type PRState struct {
CapHitAt time.Time `json:"cap_hit_at,omitempty"`
// NeedsHumanNoted is set once the needs-human note was posted.
NeedsHumanNoted bool `json:"needs_human_noted,omitempty"`
// Pending is the reply check owed for the last kick that carried
// captured rules (hivecommons/hive#19048).
Pending *PendingReply `json:"pending,omitempty"`
}

// PendingReply is the rules a kick promised to be named in the agent's reply.
type PendingReply struct {
RuleIDs []string `json:"rule_ids"`
KickedAt time.Time `json:"kicked_at"`
}

// ReplyGrace is how long the agent has to reply after a kick before a missing
// reply counts as a failure to name the rules.
const ReplyGrace = 6 * time.Hour

// Reply check verdicts.
const (
ReplyWaiting = "waiting"
ReplyOK = "ok"
ReplyMissing = "missing"
)

// CheckReply judges the agent's replies against a pending check. Only replies
// posted after the kick count. Every rule id must be named in them; with no
// reply yet the check waits until ReplyGrace has passed. Missing returns the
// ids not named.
func CheckReply(p PendingReply, replies []github.FeedbackMessage, now time.Time) (string, []string) {
var text strings.Builder
replied := false
for _, r := range replies {
if r.CreatedAt.After(p.KickedAt) {
replied = true
text.WriteString(r.Body + "\n")
}
}
if !replied {
if now.Sub(p.KickedAt) < ReplyGrace {
return ReplyWaiting, nil
}
return ReplyMissing, p.RuleIDs
}
var missing []string
for _, id := range p.RuleIDs {
if !regexp.MustCompile(`(?i)\b` + regexp.QuoteMeta(id) + `\b`).MatchString(text.String()) {
missing = append(missing, id)
}
}
if len(missing) > 0 {
return ReplyMissing, missing
}
return ReplyOK, nil
}

// RecordPendingReply notes that the kick at now carried the rule ids, so the
// reply is checked on later passes. Ids of an unanswered earlier kick stay.
func (s *Store) RecordPendingReply(key string, ids []string, now time.Time) error {
if len(ids) == 0 {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
st := s.get(key)
merged := append([]string(nil), ids...)
if st.Pending != nil {
for _, id := range st.Pending.RuleIDs {
dup := false
for _, m := range merged {
dup = dup || m == id
}
if !dup {
merged = append(merged, id)
}
}
}
sort.Strings(merged)
st.Pending = &PendingReply{RuleIDs: merged, KickedAt: now}
return s.saveLocked(now)
}

// PendingReplyFor returns the pending reply check for key, if any.
func (s *Store) PendingReplyFor(key string) (PendingReply, bool) {
s.mu.Lock()
defer s.mu.Unlock()
if st := s.PRs[key]; st != nil && st.Pending != nil {
return *st.Pending, true
}
return PendingReply{}, false
}

// ClearPendingReply drops the pending check once it was resolved or raised.
func (s *Store) ClearPendingReply(key string, now time.Time) error {
s.mu.Lock()
defer s.mu.Unlock()
st := s.PRs[key]
if st == nil || st.Pending == nil {
return nil
}
st.Pending = nil
return s.saveLocked(now)
}

// ReplyCheckNote is the needs-human note posted when the agent's reply did not
// name the rules the hive captured.
func ReplyCheckNote(missing []string) string {
return github.FeedbackNeedsHumanMarker + "\nThe hive recorded reviewer-stated rules (" + strings.Join(missing, ", ") +
") in its knowledge, but the agent's reply on this PR does not name them. A person needs to check that the reviewer's rule was acknowledged.\n"
}

// Store persists PRState per "owner/repo#number".
Expand Down
49 changes: 49 additions & 0 deletions src/pkg/reviewfeedback/reviewfeedback_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -307,3 +307,52 @@ func TestBuildBrief_TruncatesLongBodiesAndShowsThreadlessInlineItem(t *testing.T
t.Fatal("thread header must carry path:line and message url")
}
}

func TestCheckReply(t *testing.T) {
kicked := time.Now().Add(-time.Hour)
p := PendingReply{RuleIDs: []string{"R-3"}, KickedAt: kicked}
reply := func(body string, at time.Time) []github.FeedbackMessage {
return []github.FeedbackMessage{{ID: "conversation:1", Body: body, CreatedAt: at}}
}
cases := []struct {
name string
replies []github.FeedbackMessage
now time.Time
want string
}{
{"names the rule", reply("Captured as R-3.", kicked.Add(time.Minute)), time.Now(), ReplyOK},
{"names another rule", reply("Captured as R-30.", kicked.Add(time.Minute)), time.Now(), ReplyMissing},
{"does not name it", reply("Fixed.", kicked.Add(time.Minute)), time.Now(), ReplyMissing},
{"older reply does not count", reply("R-3", kicked.Add(-time.Minute)), time.Now(), ReplyWaiting},
{"no reply within grace", nil, time.Now(), ReplyWaiting},
{"no reply after grace", nil, kicked.Add(ReplyGrace + time.Minute), ReplyMissing},
}
for _, tc := range cases {
if got, _ := CheckReply(p, tc.replies, tc.now); got != tc.want {
t.Errorf("%s: verdict = %s, want %s", tc.name, got, tc.want)
}
}
}

func TestStorePendingReplyPersistsAndMerges(t *testing.T) {
dir := t.TempDir()
s, _ := Open(dir)
now := time.Now()
if err := s.RecordPendingReply("o/r#1", []string{"R-2"}, now); err != nil {
t.Fatal(err)
}
if err := s.RecordPendingReply("o/r#1", []string{"R-1"}, now.Add(time.Minute)); err != nil {
t.Fatal(err)
}
s2, _ := Open(dir)
p, ok := s2.PendingReplyFor("o/r#1")
if !ok || strings.Join(p.RuleIDs, ",") != "R-1,R-2" {
t.Fatalf("pending = %+v ok=%v", p, ok)
}
if err := s2.ClearPendingReply("o/r#1", now); err != nil {
t.Fatal(err)
}
if _, ok := s2.PendingReplyFor("o/r#1"); ok {
t.Fatal("pending not cleared")
}
}
Loading