From 5c8dac94465437205fdfbf2b8862242b6f74e21a Mon Sep 17 00:00:00 2001 From: Danathar Date: Sun, 8 Nov 2026 12:43:53 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20[v6]=20contributor-first:=20a=20?= =?UTF-8?q?relay=20at=20its=20task=20limit=20does=20not=20cover=20a=20lane?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RoleCoverage counted a relay as covering its role even when task selection refused it work because its identity had used up its tier's max_per_hour or max_per_day. The relay stays connected and is refused every ready with hourly_limit / daily_limit until the window rolls, so the governor kept the resident agent's kicks withheld while nobody worked the lane. Same class as the standby / quota-hold gaps #17233 handled. - Factor the hourly/daily rule out of selectTaskPass into tierRateLimitReached (hub.tier_limits[tier], caps <= 0 unlimited, hourly checked first). Selection keeps its log lines, decision text and refusal reasons. - RoleCoverage skips a candidate whose identity is at either cap, using the same function, after h.mu is released (rateMu only; no selectMu). - Docs: RoleCoverage comment, governor contributor-first header, and the contributor-first coverage list in agent-configuration.md. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- .../fixed-limit-reached-relay-not-cover.md | 1 + src/docs/agent-configuration.md | 9 +- src/pkg/dashboard/contribute_coverage.go | 30 +++- src/pkg/dashboard/contribute_coverage_test.go | 140 ++++++++++++++++++ src/pkg/dashboard/contribute_select.go | 83 ++++++++--- src/pkg/governor/contributor_first.go | 4 +- 6 files changed, 235 insertions(+), 32 deletions(-) create mode 100644 changelog.d/fixed-limit-reached-relay-not-cover.md diff --git a/changelog.d/fixed-limit-reached-relay-not-cover.md b/changelog.d/fixed-limit-reached-relay-not-cover.md new file mode 100644 index 0000000000..dbe53e4bdd --- /dev/null +++ b/changelog.d/fixed-limit-reached-relay-not-cover.md @@ -0,0 +1 @@ +- Contributor-first lanes are no longer left unworked by a relay that has used up its hourly or daily task allowance. A relay at its tier's `max_per_hour` or `max_per_day` is refused work until the window rolls, but it still counted as covering its role, so the governor kept the resident agent's kicks withheld while nobody worked the lane (for the daily cap, that can be most of a day). The hub now leaves such a relay out of coverage, using the same limit check that refuses it work. diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index 7704cf0ffe..7b93abfcba 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -1384,9 +1384,16 @@ A relay counts as covering the role only when all of these hold: tells the hub (`quota_hold_v1`, see [the contributor relay guide](contributor-relay.md#a-held-relay-tells-the-hub-quota_hold_v1)); an older relay is still counted while held; +- it has not used up its hourly or daily task allowance + (`hub.tier_limits..max_per_hour` / `max_per_day` for the + contributor's trust tier). The hub refuses such a relay work + (`hourly_limit` / `daily_limit`) until the window rolls, so it covers + nothing in the meantime. Coverage and task assignment use the same check, + so they always agree on which limits apply and how they are counted; - contributions are not suspended (`hub.contribute_suspended`). -A relay that is busy on a task still counts. +A relay that is busy on a task still counts, including one holding its +tier's `max_concurrent` tasks. ```yaml agents: diff --git a/src/pkg/dashboard/contribute_coverage.go b/src/pkg/dashboard/contribute_coverage.go index 25c17d074b..a4e7d037d4 100644 --- a/src/pkg/dashboard/contribute_coverage.go +++ b/src/pkg/dashboard/contribute_coverage.go @@ -31,15 +31,22 @@ import ( // Observed on 2026-11-08: a relay held at 15% of its weekly window kept // the scanner lane "covered" while nobody worked it. A relay too old to // send quota_hold is still counted, as before; +// - its identity has not used up its hourly or daily task allowance +// (hub.tier_limits max_per_hour / max_per_day for its trust tier). Task +// selection refuses such a relay with hourly_limit / daily_limit until the +// window rolls, so it takes nothing in the meantime. The check is +// tierRateLimitReached, the same function selection refuses with, so the +// two can never disagree about which limits apply or how they are counted; // - the operator has not suspended contributions (hub.contribute_suspended). // The suspend switch refuses work to every relay while their sockets stay // up, so a connected relay is then covering nothing; reporting it as // coverage would turn "pause contributors" into "pause the lane". // // A relay that is busy on a task still counts — it is draining the lane, -// which is the point. Standby and quota hold do not wait for the task to end: -// both mean the relay will not take the lane's NEXT item, which is what the -// governor is deciding whether to withhold. Whether the relay is ALLOWED the role (trust tier, +// which is the point. Standby, quota hold and a used-up allowance do not wait +// for the task to end: each means the relay will not take the lane's NEXT +// item, which is what the governor is deciding whether to withhold. (Holding +// max_concurrent tasks is different: that relay is busy, and counts.) Whether the relay is ALLOWED the role (trust tier, // grants, the delegatable list) was settled when the hub set c.role; nothing // is re-derived here. func (h *ContributeWSHub) RoleCoverage(now time.Time) map[string]governor.ContributorCoverage { @@ -47,7 +54,7 @@ func (h *ContributeWSHub) RoleCoverage(now time.Time) map[string]governor.Contri return nil } type candidate struct { - role, login, identity string + role, login, identity, tier string } var candidates []candidate h.mu.RLock() @@ -58,26 +65,33 @@ func (h *ContributeWSHub) RoleCoverage(now time.Time) map[string]governor.Contri c.mu.Lock() role := normalizeAgentRole(c.role) fresh := now.Sub(c.lastPong) <= wsHeartbeatTimeout - login := "" + login, tier := "", "" if c.profile != nil { login = strings.TrimSpace(c.profile.GitHubUsername) + tier = c.profile.TrustTier } notTakingWork := c.standbyDeclared || len(c.standby) > 0 || c.quotaHeld c.mu.Unlock() if role == "" || !fresh || login == "" || notTakingWork { continue } - candidates = append(candidates, candidate{role: role, login: login, identity: identityOf(c)}) + candidates = append(candidates, candidate{role: role, login: login, identity: identityOf(c), tier: tier}) } h.mu.RUnlock() - // The failure-streak ledger has its own lock; consult it with h.mu - // released so the two are never nested. + // The failure-streak ledger and the assignment-rate windows each have + // their own lock (failureStreakMu, rateMu); consult them with h.mu + // released so neither is ever nested inside it. The rate check is a scan + // of at most a day of assignment stamps for one identity, once per + // candidate per governor evaluation, and never takes selectMu. byRole := make(map[string]map[string]struct{}) for _, cand := range candidates { if paused, _, _ := h.contributorFailureStreakActive(cand.identity, now); paused { continue } + if h.tierRateLimitReached(cand.identity, cand.tier, now).Reason != "" { + continue + } if byRole[cand.role] == nil { byRole[cand.role] = make(map[string]struct{}) } diff --git a/src/pkg/dashboard/contribute_coverage_test.go b/src/pkg/dashboard/contribute_coverage_test.go index 37d099185f..4d89edf844 100644 --- a/src/pkg/dashboard/contribute_coverage_test.go +++ b/src/pkg/dashboard/contribute_coverage_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/governor" standbypkg "github.com/hivecommons/hive/pkg/standby" ) @@ -214,3 +215,142 @@ func TestRoleCoverage_RejectedStandbyDeclarationDoesNotCover(t *testing.T) { t.Fatalf("rejected standby relay counted as cover: RoleCoverage = %+v, want nil", got) } } + +// rateLimitedCoverageHub builds a hub whose "contributor" tier has the given +// limits and seeds identityOf(relay) with one assignment stamp per age, as +// recordAssignment does when selection hands a task out. +func rateLimitedCoverageHub(t *testing.T, limits config.TierRate, relay *ContributorConnection, now time.Time, ages ...time.Duration) (*ContributeWSHub, *Server) { + t.Helper() + hub, s := covK2Hub(t) + s.deps.Config.Hub.TierLimits = map[string]config.TierRate{"contributor": limits} + for _, age := range ages { + hub.recordAssignment(identityOf(relay), now.Add(-age)) + } + hub.mu.Lock() + hub.connections[relay.profile.GitHubUsername] = relay + hub.mu.Unlock() + return hub, s +} + +// selectionRefusal asks real task selection for work on behalf of relay's +// identity and returns the task_unavailable reason. The probe connection has +// no role, so the answer is about the identity's limits, not role policy. +func selectionRefusal(t *testing.T, hub *ContributeWSHub, s *Server, relay *ContributorConnection) string { + t.Helper() + oneActionableIssueN(s, 0) + msg := hub.selectTask(&ContributorConnection{profile: relay.profile, lastPong: time.Now()}) + if msg == nil || msg.Type != "task_unavailable" { + t.Fatalf("setup: selection must refuse this identity, got %+v", msg) + } + return msg.Reason +} + +// A relay that has been handed its tier's max_per_hour tasks in the last hour +// is refused work with hourly_limit until the window rolls. It is connected +// and acting as the role, but it will not take the lane's next item, so it +// must not keep the resident agent withheld. +func TestRoleCoverage_HourlyLimitReachedDoesNotCover(t *testing.T) { + now := time.Now() + relay := coverageConn("alice", "quality", now) + hub, s := rateLimitedCoverageHub(t, config.TierRate{MaxPerHour: 2, MaxPerDay: 50, MaxConcurrent: 5}, + relay, now, 10*time.Minute, 20*time.Minute) + + if got := selectionRefusal(t, hub, s, relay); got != taskUnavailableHourlyLimit { + t.Fatalf("setup: selection refused with %q, want %q", got, taskUnavailableHourlyLimit) + } + if got := hub.RoleCoverage(now); got != nil { + t.Fatalf("relay at its hourly limit counted as cover: RoleCoverage = %+v, want nil", got) + } +} + +// The same for the daily cap: no assignment in the last hour, but the tier's +// max_per_day in the last 24 hours, so selection refuses with daily_limit. +func TestRoleCoverage_DailyLimitReachedDoesNotCover(t *testing.T) { + now := time.Now() + relay := coverageConn("alice", "scanner", now) + hub, s := rateLimitedCoverageHub(t, config.TierRate{MaxPerHour: 10, MaxPerDay: 3, MaxConcurrent: 5}, + relay, now, 2*time.Hour, 5*time.Hour, 20*time.Hour) + + if got := selectionRefusal(t, hub, s, relay); got != taskUnavailableDailyLimit { + t.Fatalf("setup: selection refused with %q, want %q", got, taskUnavailableDailyLimit) + } + if got := hub.RoleCoverage(now); got != nil { + t.Fatalf("relay at its daily limit counted as cover: RoleCoverage = %+v, want nil", got) + } +} + +// A relay one task short of both caps still covers: only a used-up allowance +// takes a relay out. Stamps older than the day window do not count. Bob, in +// the same tier and lane, is at the hourly cap and is the only one dropped. +func TestRoleCoverage_UnderRateLimitsCovers(t *testing.T) { + now := time.Now() + alice := coverageConn("alice", "quality", now) + bob := coverageConn("bob", "quality", now) + // alice: 1 of 2 this hour, 2 of 3 today (the 25h-old stamp has rolled off). + hub, _ := rateLimitedCoverageHub(t, config.TierRate{MaxPerHour: 2, MaxPerDay: 3, MaxConcurrent: 5}, + alice, now, 10*time.Minute, 3*time.Hour, 25*time.Hour) + hub.recordAssignment(identityOf(bob), now.Add(-5*time.Minute)) + hub.recordAssignment(identityOf(bob), now.Add(-15*time.Minute)) + hub.mu.Lock() + hub.connections["bob"] = bob + hub.mu.Unlock() + + want := map[string]governor.ContributorCoverage{"quality": {Contributors: []string{"alice"}}} + if got := hub.RoleCoverage(now); !reflect.DeepEqual(got, want) { + t.Fatalf("RoleCoverage = %+v, want %+v (alice under both caps, bob at the hourly cap)", got, want) + } +} + +// tierRateLimitReached is the single rule selection and RoleCoverage share. +func TestTierRateLimitReached(t *testing.T) { + now := time.Date(2026, 11, 8, 12, 0, 0, 0, time.UTC) + cases := []struct { + name string + limits map[string]config.TierRate + tier string + ages []time.Duration + want tierRateLimitHit + }{ + {name: "under both caps", limits: map[string]config.TierRate{"contributor": {MaxPerHour: 2, MaxPerDay: 3}}, + tier: "contributor", ages: []time.Duration{time.Minute, 2 * time.Hour}}, + {name: "at the hourly cap", limits: map[string]config.TierRate{"contributor": {MaxPerHour: 2, MaxPerDay: 50}}, + tier: "contributor", ages: []time.Duration{time.Minute, 30 * time.Minute}, + want: tierRateLimitHit{Reason: taskUnavailableHourlyLimit, Count: 2, Max: 2}}, + {name: "at the daily cap", limits: map[string]config.TierRate{"contributor": {MaxPerHour: 10, MaxPerDay: 2}}, + tier: "contributor", ages: []time.Duration{2 * time.Hour, 23 * time.Hour}, + want: tierRateLimitHit{Reason: taskUnavailableDailyLimit, Count: 2, Max: 2}}, + {name: "both caps reports hourly first", limits: map[string]config.TierRate{"contributor": {MaxPerHour: 1, MaxPerDay: 1}}, + tier: "contributor", ages: []time.Duration{time.Minute}, + want: tierRateLimitHit{Reason: taskUnavailableHourlyLimit, Count: 1, Max: 1}}, + {name: "stamps past the day window do not count", limits: map[string]config.TierRate{"contributor": {MaxPerHour: 1, MaxPerDay: 1}}, + tier: "contributor", ages: []time.Duration{25 * time.Hour}}, + {name: "zero caps are unlimited", limits: map[string]config.TierRate{"advisor": {}}, + tier: "advisor", ages: []time.Duration{time.Minute, 2 * time.Minute, 3 * time.Minute}}, + {name: "a tier with no row is unlimited", limits: map[string]config.TierRate{"contributor": {MaxPerHour: 1, MaxPerDay: 1}}, + tier: "custom", ages: []time.Duration{time.Minute, 2 * time.Minute}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + hub, s := covK2Hub(t) + s.deps.Config.Hub.TierLimits = tc.limits + for _, age := range tc.ages { + hub.recordAssignment("c-alice", now.Add(-age)) + } + if got := hub.tierRateLimitReached("c-alice", tc.tier, now); got != tc.want { + t.Fatalf("tierRateLimitReached = %+v, want %+v", got, tc.want) + } + }) + } + + // No server or config behind the hub: nothing to enforce, never a panic. + bare := NewContributeWSHub(slog.Default(), nil) + t.Cleanup(bare.Close) + bare.recordAssignment("c-alice", now) + if got := bare.tierRateLimitReached("c-alice", "contributor", now); got != (tierRateLimitHit{}) { + t.Fatalf("hub without a server: tierRateLimitReached = %+v, want zero", got) + } + var none *ContributeWSHub + if got := none.tierRateLimitReached("c-alice", "contributor", now); got != (tierRateLimitHit{}) { + t.Fatalf("nil hub: tierRateLimitReached = %+v, want zero", got) + } +} diff --git a/src/pkg/dashboard/contribute_select.go b/src/pkg/dashboard/contribute_select.go index 5b46bf9512..b0c804acd3 100644 --- a/src/pkg/dashboard/contribute_select.go +++ b/src/pkg/dashboard/contribute_select.go @@ -162,6 +162,48 @@ func (h *ContributeWSHub) rateWindowCounts(identity string, now time.Time) (hour return hour, day } +// tierRateLimitHit says which assignment-rate cap an identity has used up: +// Reason is taskUnavailableHourlyLimit or taskUnavailableDailyLimit, Count is +// the assignments counted in that window and Max the cap. A zero value means +// the identity may be handed another task. +type tierRateLimitHit struct { + Reason string + Count, Max int +} + +// tierRateLimitReached is the one rule for "this identity has used up its +// hourly or daily task allowance" (#2566). selectTaskPass refuses work with it, +// and RoleCoverage uses it so a relay that selection refuses for this reason +// does not count as contributor-first cover. +// +// The limits are hub.tier_limits[tier] for the connection's trust tier, and +// nothing else: config load fills in the shipped row for each built-in tier +// the operator did not set (config/defaults.go), there is no per-contributor +// override (ContributorProfile.RateLimits is stored but never enforced), and a +// tier with no row is unlimited. A cap of 0 or less is unlimited. The hourly +// cap is checked first, so a relay at both caps reports hourly_limit, as +// selection always has. +// +// It takes rateMu (through rateWindowCounts) and reads the config without a +// lock, as selection does. The caller must not hold rateMu. +func (h *ContributeWSHub) tierRateLimitReached(identity, tier string, now time.Time) tierRateLimitHit { + if h == nil || h.server == nil || h.server.deps == nil || h.server.deps.Config == nil { + return tierRateLimitHit{} + } + limits, ok := h.server.deps.Config.Hub.TierLimits[tier] + if !ok || (limits.MaxPerHour <= 0 && limits.MaxPerDay <= 0) { + return tierRateLimitHit{} + } + hourCount, dayCount := h.rateWindowCounts(identity, now) + if limits.MaxPerHour > 0 && hourCount >= limits.MaxPerHour { + return tierRateLimitHit{Reason: taskUnavailableHourlyLimit, Count: hourCount, Max: limits.MaxPerHour} + } + if limits.MaxPerDay > 0 && dayCount >= limits.MaxPerDay { + return tierRateLimitHit{Reason: taskUnavailableDailyLimit, Count: dayCount, Max: limits.MaxPerDay} + } + return tierRateLimitHit{} +} + // recordAssignment appends an assignment timestamp for the identity. Called once // per task_assign actually shipped, so the rate windows count tasks HANDED OUT // (matching max_concurrent's semantics and the max_tasks_per_hour/day naming), not @@ -590,29 +632,28 @@ func (h *ContributeWSHub) selectTaskPass(c *ContributorConnection, skippedUnmint strconv.Itoa(limits.MaxConcurrent)) return h.taskUnavailable(taskUnavailableConcurrencyLimit) } - if limits.MaxPerHour > 0 || limits.MaxPerDay > 0 { - hourCount, dayCount := h.rateWindowCounts(identityOf(c), time.Now()) - if limits.MaxPerHour > 0 && hourCount >= limits.MaxPerHour { - h.logger.Warn("[contribute-ws] refusing task: hourly rate limit reached", - "username", identityOf(c), "tier", tier, - "assigned_last_hour", hourCount, "max_per_hour", limits.MaxPerHour) - h.recordDecision(decisionUsername(c), decisionRefused, "", "", 0, - "hourly rate limit for tier "+tier+": "+strconv.Itoa(hourCount)+ - " assigned in the last hour, max "+strconv.Itoa(limits.MaxPerHour)) - return h.taskUnavailable(taskUnavailableHourlyLimit) - } - if limits.MaxPerDay > 0 && dayCount >= limits.MaxPerDay { - h.logger.Warn("[contribute-ws] refusing task: daily rate limit reached", - "username", identityOf(c), "tier", tier, - "assigned_last_day", dayCount, "max_per_day", limits.MaxPerDay) - h.recordDecision(decisionUsername(c), decisionRefused, "", "", 0, - "daily rate limit for tier "+tier+": "+strconv.Itoa(dayCount)+ - " assigned in the last day, max "+strconv.Itoa(limits.MaxPerDay)) - return h.taskUnavailable(taskUnavailableDailyLimit) - } - } } } + // The hourly/daily caps live in tierRateLimitReached, which RoleCoverage + // shares, so a relay refused here is never reported as covering its role. + switch hit := h.tierRateLimitReached(identityOf(c), tier, time.Now()); hit.Reason { + case taskUnavailableHourlyLimit: + h.logger.Warn("[contribute-ws] refusing task: hourly rate limit reached", + "username", identityOf(c), "tier", tier, + "assigned_last_hour", hit.Count, "max_per_hour", hit.Max) + h.recordDecision(decisionUsername(c), decisionRefused, "", "", 0, + "hourly rate limit for tier "+tier+": "+strconv.Itoa(hit.Count)+ + " assigned in the last hour, max "+strconv.Itoa(hit.Max)) + return h.taskUnavailable(taskUnavailableHourlyLimit) + case taskUnavailableDailyLimit: + h.logger.Warn("[contribute-ws] refusing task: daily rate limit reached", + "username", identityOf(c), "tier", tier, + "assigned_last_day", hit.Count, "max_per_day", hit.Max) + h.recordDecision(decisionUsername(c), decisionRefused, "", "", 0, + "daily rate limit for tier "+tier+": "+strconv.Itoa(hit.Count)+ + " assigned in the last day, max "+strconv.Itoa(hit.Max)) + return h.taskUnavailable(taskUnavailableDailyLimit) + } // #6450: pause claims for an identity whose recent assignments all died // within seconds — a dying agent runtime. Checked after the operator gates diff --git a/src/pkg/governor/contributor_first.go b/src/pkg/governor/contributor_first.go index 8a922bcc38..9013c30a93 100644 --- a/src/pkg/governor/contributor_first.go +++ b/src/pkg/governor/contributor_first.go @@ -21,8 +21,8 @@ import ( // // contributor_first inverts the order for one lane. While at least one // connected, non-stale, non-failure-paused relay is acting as the role and -// taking work (not in standby, not quota-held; see RoleCoverage in -// pkg/dashboard), and the operator has not suspended contributions, the +// taking work (not in standby, not quota-held, not at its hourly or daily +// task limit; see RoleCoverage in pkg/dashboard), and the operator has not suspended contributions, the // governor WITHHOLDS the resident agent's kicks — at exactly the gates where it // withholds them for an exhausted budget. The relay is the primary worker; the // resident agent is the fallback, and it resumes on its normal cadence once no