Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-14668.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- A relay `fix-pr` task that fails right after dispatch is no longer reported as delivered: the stage-072 nudge comment says the relay could not take the task (with its reason), the fix is requeued with backoff and retried at the relay's next task boundary (three attempts before the re-nudge / adoption path takes over), and a conflicting contributor PR drops its stale `hive:unauthorized` mark so the conflict is the row's blocker ([#14668](https://github.com/hivecommons/hive/issues/14668)).
2 changes: 2 additions & 0 deletions src/docs/contributor-relay.md
Original file line number Diff line number Diff line change
Expand Up @@ -1221,6 +1221,8 @@ The one place a declaration DOES shape dispatch is the stage-072 contributor nud

The hub records the mode it used in the nudge comment's marker (`mode=spawn|preempt|queued|comment`) and exposes queue depth / in-flight as `fix_queue` on the contributor profile API.

**When the relay fails the task.** `task_failed` for a fix's `task_id` does not settle it as done: its PRs go back in the hub's queue (same place by age) and are not interrupted back in for 5 minutes, though the relay's next `ready` serves them at once. After three failed dispatches a PR leaves the fix queue and the stage engine's re-nudge / adoption path takes over. A dispatch the relay fails within one second of the nudge is not reported as delivered: the nudge comment reads `mode=queued`, says the relay could not take the task (quoting the relay's `message`/`reason`), and the stage log carries `relay_failed=true` (hivecommons/hive#14668).

#### What the operator sees (Operations → Connected contributor agents)

Each row of the "Connected contributor agents (ClankeR)" list on `/contribute/operations` reads the same declaration back (hivecommons/hive#12502), so an operator can check whether the rework the stage labels promise is actually reaching the agent. The row is fed by `/api/contribute/fleet`; each `clankers[]` entry carries:
Expand Down
8 changes: 5 additions & 3 deletions src/docs/labels-and-control-signals.md
Original file line number Diff line number Diff line change
Expand Up @@ -1132,9 +1132,11 @@ When a clanker-authored contributor PR goes red the hive does not just relabel i
focus-window rule; this ordering applies at task-boundary time.
7. Capacity ≥ 2: fixes are dispatched immediately as parallel sub-sessions,
up to the advertised count so the primary session keeps going and is
never preempted; further failures queue, and each finished fix session
(`task_complete`/`task_failed` for the fix's `task_id`) frees a slot that
is refilled automatically with the next queued `fix-pr`.
never preempted; further failures queue, and each completed fix session
(`task_complete` for the fix's `task_id`) frees a slot that is refilled
automatically with the next queued `fix-pr`. A `task_failed` requeues the
fix with backoff instead (three attempts, then the re-nudge / adoption
path; see [contributor-relay.md § fix-pr tasks](contributor-relay.md#fix-pr-tasks)).

All seven steps ship with the stage engine and the contribute hub
(hivecommons/hive#12436).
Expand Down
150 changes: 150 additions & 0 deletions src/pkg/dashboard/contribute_fixqueue.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package dashboard

import (
Expand Down Expand Up @@ -43,6 +43,18 @@
// taskUnavailableFixInFlight is the task_unavailable reason when a
// contributor's queued fixes exist but every slot is busy: new work waits.
taskUnavailableFixInFlight = "fix_pr_in_flight"

// fixTaskAckGrace is how long Nudge waits after an interrupt dispatch for
// the relay to refuse it before reporting the nudge as delivered; relays
// that cannot take the task fail it within ~100ms (hivecommons/hive#14668).
fixTaskAckGrace = time.Second
// fixTaskRetryBackoff keeps a failed fix from being interrupted straight
// back into a relay that may be mid-turn; a task boundary still serves it.
fixTaskRetryBackoff = 5 * time.Minute
// fixTaskMaxAttempts is how many failed dispatches a PR gets before it
// leaves the fix queue and the stage engine's re-nudge / adoption path
// takes over.
fixTaskMaxAttempts = 3
)

// WSFixPR is one red PR inside a fix-pr task_assign.
Expand Down Expand Up @@ -257,8 +269,15 @@
h.rememberFixOwner(author, item.Key())
out.Mode = "queued"
if online {
failed := h.beginFixWait()
defer h.endFixWait()
if msg := h.dispatchFixes(author, conns, false); msg != nil {
out.Mode = msg.Mode
if reason, refused := h.awaitFixTaskRefusal(msg.TaskID, failed); refused {
out.Mode = "queued"
out.RelayFailed = true
out.RelayError = reason
}
}
}
st := sched.Status(author, fixqueue.Capacity{Max: 1})
Expand Down Expand Up @@ -464,6 +483,137 @@
return true
}

// failFixTask settles a fix-pr task the relay reported failed: its PRs go
// back in the queue with backoff, and a PR that has failed
// fixTaskMaxAttempts times leaves the queue for the stage engine's re-nudge
// and adoption path. The freed slot is not refilled here — that would hand
// the same fix straight back. It returns false when msg.TaskID is not a fix
// task on this connection.
func (h *ContributeWSHub) failFixTask(c *ContributorConnection, msg WSMessage) bool {
c.mu.Lock()
_, ok := c.fixTasks[msg.TaskID]
if ok {
delete(c.fixTasks, msg.TaskID)
delete(c.fixTasksAt, msg.TaskID)
}
c.mu.Unlock()
if !ok {
return false
}
reason := relayFailureReason(msg)
if c.profile == nil {
h.logger.Info("[contribute-ws] fix-pr task failed", "username", identityOf(c), "task", msg.TaskID, "reason", reason)
return true
}
user := c.profile.GitHubUsername
sched := h.fixQueue()
requeued := sched.Requeue(user, msg.TaskID, fixTaskRetryBackoff)
if len(requeued) == 0 {
h.logger.Info("[contribute-ws] fix-pr task failed", "username", user, "task", msg.TaskID, "reason", reason)
}
for _, it := range requeued {
if it.Attempts >= fixTaskMaxAttempts {
sched.Remove(user, it.Repo, it.Number)
h.forgetFixOwner(user, it.Key())
h.logger.Warn("[contribute-ws] fix-pr task failed; giving up on relay delivery", "username", user, "task", msg.TaskID, "pr", it.Key(), "attempts", it.Attempts, "reason", reason)
continue
}
h.logger.Info("[contribute-ws] fix-pr task failed; requeued", "username", user, "task", msg.TaskID, "pr", it.Key(), "attempts", it.Attempts, "retry_after", it.NotBefore, "reason", reason)
}
h.pulseFixFailed(msg.TaskID, reason)
return true
}

// relayFailureReason is the relay's own words for a task_failed.
func relayFailureReason(msg WSMessage) string {
for _, r := range []string{msg.Message, msg.Reason, msg.FailureKind} {
if r = strings.TrimSpace(r); r != "" {
return r
}
}
return ""
}

// beginFixWait registers a Nudge waiting to learn whether its dispatch is
// refused and returns the channel closed at the next fix-pr failure. Call it
// BEFORE dispatching so a failure racing the dispatch is not missed; pair it
// with endFixWait. Failures are only remembered while someone waits.
func (h *ContributeWSHub) beginFixWait() <-chan struct{} {
h.fixQueueMu.Lock()
defer h.fixQueueMu.Unlock()
h.fixWaiters++
return h.fixFailedSignalLocked()
}

func (h *ContributeWSHub) endFixWait() {
h.fixQueueMu.Lock()
defer h.fixQueueMu.Unlock()
if h.fixWaiters > 0 {
h.fixWaiters--
}
if h.fixWaiters == 0 {
h.fixFailures = nil
}
}

func (h *ContributeWSHub) fixFailedSignalLocked() chan struct{} {
if h.fixFailedSig == nil {
h.fixFailedSig = make(chan struct{})
}
return h.fixFailedSig
}

// pulseFixFailed records why taskID failed (while a Nudge waits) and wakes
// every waiter.
func (h *ContributeWSHub) pulseFixFailed(taskID, reason string) {
h.fixQueueMu.Lock()
defer h.fixQueueMu.Unlock()
if h.fixWaiters > 0 {
if h.fixFailures == nil {
h.fixFailures = map[string]string{}
}
h.fixFailures[taskID] = reason
}
if h.fixFailedSig != nil {
close(h.fixFailedSig)
h.fixFailedSig = nil
}
}

// takeFixFailure reports (and forgets) whether taskID failed, and why.
func (h *ContributeWSHub) takeFixFailure(taskID string) (string, bool) {
h.fixQueueMu.Lock()
defer h.fixQueueMu.Unlock()
reason, ok := h.fixFailures[taskID]
delete(h.fixFailures, taskID)
return reason, ok
}

// awaitFixTaskRefusal waits up to the ack grace for the relay to fail the
// fix-pr task it was just handed. refused=true means it did, so the nudge
// was not delivered as a task.
func (h *ContributeWSHub) awaitFixTaskRefusal(taskID string, failed <-chan struct{}) (reason string, refused bool) {
grace := h.fixAckGrace
if grace <= 0 {
grace = fixTaskAckGrace
}
timer := time.NewTimer(grace)
defer timer.Stop()
for {
if r, ok := h.takeFixFailure(taskID); ok {
return r, true
}
select {
case <-failed:
h.fixQueueMu.Lock()
failed = h.fixFailedSignalLocked()
h.fixQueueMu.Unlock()
case <-timer.C:
return h.takeFixFailure(taskID)
}
}
}

// FixQueueStatus is the contributor's fix-queue view for the profile API.
func (h *ContributeWSHub) FixQueueStatus(username string) fixqueue.Status {
conns, _ := h.liveConnectionsFor(username, time.Now())
Expand Down
61 changes: 61 additions & 0 deletions src/pkg/dashboard/contribute_fixqueue_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -275,3 +275,64 @@ func TestFixPRInstructionCoalesced(t *testing.T) {
}
}
}

// hivecommons/hive#14668: a relay that fails the fix-pr task right after
// dispatch must not be reported as reached. The nudge reads "queued" with the
// relay's reason, the PR waits in the queue (no immediate re-interrupt), the
// next task boundary retries it, and after fixTaskMaxAttempts failures it
// leaves the queue for the stage engine's re-nudge / adoption path.
func TestFixQueueInstantRelayFailureIsNotDelivered(t *testing.T) {
hub, _ := covK2Hub(t)
hub.fixAckGrace = time.Minute
c := fixConn(hub, "alice", fixCaps(capFixPRInterrupt), time.Now())
s := &wsSession{h: hub, contributor: c}
var got []WSMessage
hub.fixDispatched = func(_ *ContributorConnection, m WSMessage) {
got = append(got, m)
if len(got) == 1 {
go s.handleTaskFailed(WSMessage{Type: "task_failed", TaskID: m.TaskID, Message: "codex: session busy"})
}
}

out := hub.Nudge(context.Background(), fixNudge("r", 1, "build"))
if !out.Online || out.Mode != "queued" || !out.RelayFailed || out.RelayError != "codex: session busy" || out.QueueDepth != 1 {
t.Fatalf("instantly failed dispatch reported as %+v, want queued + relay failure", out)
}
if len(got) != 1 {
t.Fatalf("dispatches = %d, want the failed fix not interrupted straight back", len(got))
}
if st := hub.FixQueueStatus("alice"); st.Queued != 1 || st.InFlight != 0 {
t.Fatalf("status after failure = %+v, want requeued", st)
}
if q := hub.FixQueueItems("alice"); len(q) != 1 || q[0].Attempts != 1 {
t.Fatalf("queued = %+v, want one item with 1 failed attempt", q)
}
c.mu.Lock()
tracked := len(c.fixTasks)
c.mu.Unlock()
if tracked != 0 {
t.Fatalf("failed fix task still tracked on the connection (%d)", tracked)
}

// The relay's next task boundary retries it; two more failures exhaust it.
for attempt := 2; attempt <= fixTaskMaxAttempts; attempt++ {
msg := hub.selectFixTask(c)
if msg == nil || msg.Kind != taskKindFixPR || msg.Priority != taskPriorityQueued {
t.Fatalf("attempt %d: boundary should serve the queued fix, got %+v", attempt, msg)
}
s.handleTaskFailed(WSMessage{Type: "task_failed", TaskID: msg.TaskID})
}
if st := hub.FixQueueStatus("alice"); st.Queued != 0 || st.InFlight != 0 {
t.Fatalf("status after %d failures = %+v, want the PR out of the queue", fixTaskMaxAttempts, st)
}
if msg := hub.selectFixTask(c); msg != nil {
t.Fatalf("exhausted fix served again: %+v", msg)
}
hub.fixQueueMu.Lock()
owned := len(hub.fixOwners["alice"])
waiters, failures := hub.fixWaiters, len(hub.fixFailures)
hub.fixQueueMu.Unlock()
if owned != 0 || waiters != 0 || failures != 0 {
t.Fatalf("leftover state: owners=%d waiters=%d failures=%d", owned, waiters, failures)
}
}
15 changes: 13 additions & 2 deletions src/pkg/dashboard/contribute_ws.go
Original file line number Diff line number Diff line change
Expand Up @@ -903,6 +903,16 @@ type ContributeWSHub struct {
fixSched *fixqueue.Scheduler
fixOwners map[string]map[string]bool
fixDispatched func(*ContributorConnection, WSMessage)
// fixFailedSig is closed (and replaced) whenever a relay reports a
// fix-pr task failed, waking a Nudge that is waiting to learn whether its
// dispatch was refused (hivecommons/hive#14668). fixAckGrace overrides
// that wait (zero = fixTaskAckGrace); a test hook.
// fixWaiters counts those Nudges; fixFailures (task id → relay reason)
// is kept only while one waits.
fixFailedSig chan struct{}
fixWaiters int
fixFailures map[string]string
fixAckGrace time.Duration
// stageStore is the stage-label memory the fleet view reads "PRs returned
// to this contributor" from (hivecommons/hive#12502). Nil means the
// process-wide stagelabels.Shared(); tests inject an in-memory store.
Expand Down Expand Up @@ -4156,8 +4166,9 @@ func (s *wsSession) handleTaskFailed(msg WSMessage) {
h := s.h
// hivecommons/hive#12436: a fix-pr task settles in the contributor's fix
// queue (freeing its slot and refilling it) and never touches currentTask.
if s.contributor != nil && msg.TaskID != "" && h.completeFixTask(s.contributor, msg.TaskID) {
h.logger.Info("[contribute-ws] fix-pr task failed", "username", identityOf(s.contributor), "task", msg.TaskID)
// hivecommons/hive#14668: a failed fix is requeued with backoff (retried,
// then dropped back to the stage engine), not settled as done.
if s.contributor != nil && msg.TaskID != "" && h.failFixTask(s.contributor, msg) {
return
}
if s.contributor != nil {
Expand Down
51 changes: 47 additions & 4 deletions src/pkg/fixqueue/fixqueue.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,10 @@ type Item struct {
// EnqueuedAt is when the item entered the queue (set by Enqueue; read by
// Overflow).
EnqueuedAt time.Time
// Attempts counts the dispatches of this item the relay reported failed
// (set by Requeue); NotBefore is when an interrupt may retry it.
Attempts int
NotBefore time.Time
}

// Key identifies an item.
Expand Down Expand Up @@ -225,14 +229,14 @@ func (s *Scheduler) Enqueue(user string, it Item) bool {
key := it.Key()
for _, d := range c.inflight {
if old, ok := d.items[key]; ok {
it.EnqueuedAt = old.EnqueuedAt
it.EnqueuedAt, it.Attempts, it.NotBefore = old.EnqueuedAt, old.Attempts, old.NotBefore
d.items[key] = it
return false
}
}
for i, q := range c.queue {
if q.Key() == key {
it.EnqueuedAt = q.EnqueuedAt
it.EnqueuedAt, it.Attempts, it.NotBefore = q.EnqueuedAt, q.Attempts, q.NotBefore
c.queue[i] = it
sortQueue(c.queue)
return false
Expand Down Expand Up @@ -338,17 +342,56 @@ func (s *Scheduler) CompleteDispatch(user, id string) bool {
return false
}

// Requeue settles a dispatch the relay reported failed and puts its items
// back in the queue (keeping their place by age) with Attempts incremented
// and NotBefore = now+backoff, so an interrupt does not hand the same fix
// straight back to a relay that just refused it. It returns the requeued
// items; nil when id is not in flight.
func (s *Scheduler) Requeue(user, id string, backoff time.Duration) []Item {
s.mu.Lock()
defer s.mu.Unlock()
c, ok := s.by[norm(user)]
if !ok {
return nil
}
for i, d := range c.inflight {
if d.id != id {
continue
}
c.inflight = append(c.inflight[:i], c.inflight[i+1:]...)
if d.mode == ModePreempt {
// The primary was never parked for long: no focus window.
c.preempted = false
}
out := d.public().Items
notBefore := s.opts.Now().Add(backoff)
for j := range out {
out[j].Attempts++
out[j].NotBefore = notBefore
c.queue = append(c.queue, out[j])
}
sortQueue(c.queue)
s.log("requeue", user, Dispatch{ID: d.id, Mode: d.mode, Items: out}, len(c.queue))
return out
}
return nil
}

// Dispatch tries to hand user the next fix while a primary task is running.
// It returns nil when nothing may be dispatched now: empty queue, all slots
// busy, a preemption still in flight, or the resumed primary's focus window
// has not elapsed. The returned Dispatch's Mode is spawn or preempt.
// busy, a preemption still in flight, the resumed primary's focus window
// has not elapsed, or the queue head is backing off after a failed dispatch.
// The returned Dispatch's Mode is spawn or preempt.
func (s *Scheduler) Dispatch(user string, cap Capacity) *Dispatch {
s.mu.Lock()
defer s.mu.Unlock()
c, ok := s.by[norm(user)]
if !ok || len(c.queue) == 0 || len(c.inflight) >= cap.Slots() {
return nil
}
if s.opts.Now().Before(c.queue[0].NotBefore) {
return nil
}
mode := cap.Mode()
if mode == ModePreempt && (c.preempted || s.opts.Now().Before(c.focusUntil)) {
return nil
Expand Down
Loading
Loading