diff --git a/changelog.d/fixed-writeback-cutoff-needs-prior-refusal.md b/changelog.d/fixed-writeback-cutoff-needs-prior-refusal.md new file mode 100644 index 0000000000..4be293bb2a --- /dev/null +++ b/changelog.d/fixed-writeback-cutoff-needs-prior-refusal.md @@ -0,0 +1 @@ +- Write-back backfill guard: a successful write with no 403/404 before it no longer moves a repository's backfill cut-off (`writable_at`). Only a few writers report outcomes, so the first one to succeed could come days after onboarding, and the guard then froze the stage labels of every older PR and issue the hive had been labelling all along. A `writable_at` stamped that way is cleared at startup (logged as `writeback: cleared writable_at …`) with the items it refused, and `/api/status` `writeback.repos[].writable` now reads true for a repository that never refused a write. diff --git a/src/docs/operator-reference/sweeps-and-operations.md b/src/docs/operator-reference/sweeps-and-operations.md index 2feda4ff1d..e570516e26 100644 --- a/src/docs/operator-reference/sweeps-and-operations.md +++ b/src/docs/operator-reference/sweeps-and-operations.md @@ -671,11 +671,15 @@ and the backfill banner. `/api/status` carries the same state under The gate persists `first_seen` per repo in `/data/metrics/writeback.json` (`HIVE_WRITEBACK_STATE_PATH`), and records the moment a repo first answers -2xx after 403/404s as `writable_at`. Items created before the later of the -two are never written to unsolicited while `backfill: false`. The sweep logs -one line — `writeback: skipped N pre-existing items on (backfill -off)` — and the dashboard shows "hive can now write to ``; N existing -PRs were left untouched — enable backfill to process them". +2xx after 403/404s as `writable_at`. A 2xx with no 403/404 before it does +not set `writable_at`: the repo was writable all along, so `first_seen` alone +is the cut-off (a `writable_at` an older hive stamped that way is cleared at +startup, logged as `writeback: cleared writable_at …`). Items created before +the later of the two are never written to unsolicited while +`backfill: false`. The sweep logs one line — `writeback: skipped N +pre-existing items on (backfill off)` — and the dashboard shows +"hive can now write to ``; N existing PRs were left untouched — +enable backfill to process them". ### Duplicate-write suppression diff --git a/src/pkg/writeback/gate.go b/src/pkg/writeback/gate.go index 571d44c12c..e92b89ff39 100644 --- a/src/pkg/writeback/gate.go +++ b/src/pkg/writeback/gate.go @@ -145,7 +145,10 @@ type Suppressed struct { // RepoStatus is one repo's block in the status payload. type RepoStatus struct { - FirstSeen time.Time `json:"first_seen,omitempty"` + FirstSeen time.Time `json:"first_seen,omitempty"` + // Writable is false only while a 403/404 has had no 2xx after it; a + // repo that never refused a write was writable all along and carries + // no WritableAt. Writable bool `json:"writable"` WritableAt time.Time `json:"writable_at,omitempty"` Mode RepoMode `json:"mode,omitempty"` @@ -188,7 +191,8 @@ type repoRecord struct { FirstSeen time.Time `json:"first_seen"` WritableAt time.Time `json:"writable_at,omitempty"` // LastForbidden is the last 403/404 a write to this repo got, so the - // first 2xx after it marks the moment the App gained write. + // first 2xx after it marks the moment the App gained write. A repo that + // never got one was writable all along and keeps WritableAt zero. LastForbidden time.Time `json:"last_forbidden,omitempty"` // PreExisting is the set of item numbers the backfill guard refused. PreExisting map[int]bool `json:"pre_existing,omitempty"` @@ -269,9 +273,32 @@ func New(cfg Config, deps Deps, path string) *Gate { g.deps.Logger = slog.Default() } g.load() + g.repairBackfillCutoffs() return g } +// repairBackfillCutoffs clears a writable_at that NoteWriteOutcome stamped +// on a 2xx with no 403/404 before it, before it required one: that repo was +// writable all along, so first_seen alone is the cut-off. The items the +// wrong cut-off refused are dropped too; the guard records again any that +// really predate first_seen the next time it refuses one. +func (g *Gate) repairBackfillCutoffs() { + repaired := false + for k, rec := range g.repos { + if rec == nil || rec.WritableAt.IsZero() || !rec.LastForbidden.IsZero() { + continue + } + g.deps.Logger.Info(fmt.Sprintf("writeback: cleared writable_at %s on %s (no 403/404 before it); backfill cut-off is first_seen %s", + rec.WritableAt.Format(time.RFC3339), k, rec.FirstSeen.Format(time.RFC3339))) + rec.WritableAt = time.Time{} + rec.PreExisting = nil + repaired = true + } + if repaired { + g.saveLocked() + } +} + // SetConfig swaps the policy (dashboard edits take effect on the next // decision). func (g *Gate) SetConfig(cfg Config) { @@ -384,7 +411,10 @@ func (g *Gate) ObserveRepo(repo string) { // NoteWriteOutcome records the result of a real GitHub write to repo so the // gate can detect the App gaining write: a 403/404 marks the repo as not // writable; the first 2xx after one marks WritableAt now, which the -// backfill guard uses as the cut-off for items created before it. +// backfill guard uses as the cut-off for items created before it. A 2xx +// with no 403/404 before it is not a gain: only a few writers report here, +// so the first one to succeed can come days after the hive started +// labelling the repo, and stamping it froze every older item. func (g *Gate) NoteWriteOutcome(repo string, statusCode int) { if g == nil || strings.TrimSpace(repo) == "" { return @@ -398,7 +428,7 @@ func (g *Gate) NoteWriteOutcome(repo string, statusCode int) { rec.LastForbidden = now g.saveLocked() case statusCode >= 200 && statusCode < 300: - if rec.WritableAt.IsZero() { + if rec.WritableAt.IsZero() && !rec.LastForbidden.IsZero() { rec.WritableAt = now g.saveLocked() } @@ -763,7 +793,7 @@ func (g *Gate) Snapshot() Status { if g.deps.LevelFor != nil { level = g.deps.LevelFor(k) } - rs := RepoStatus{FirstSeen: rec.FirstSeen, Writable: !rec.WritableAt.IsZero(), WritableAt: rec.WritableAt, Mode: cfg.RepoModeFor(k), Level: level, Suppressed24: per[k], PreExisting: len(rec.PreExisting)} + rs := RepoStatus{FirstSeen: rec.FirstSeen, Writable: !rec.WritableAt.IsZero() || rec.LastForbidden.IsZero(), WritableAt: rec.WritableAt, Mode: cfg.RepoModeFor(k), Level: level, Suppressed24: per[k], PreExisting: len(rec.PreExisting)} st.Repos[k] = rs if rs.PreExisting > 0 && !cfg.BackfillEnabled() { st.Banners = append(st.Banners, Banner{Repo: k, WritableAt: rec.WritableAt, Untouched: rs.PreExisting, diff --git a/src/pkg/writeback/writeback_test.go b/src/pkg/writeback/writeback_test.go index ed1a878509..436395aad0 100644 --- a/src/pkg/writeback/writeback_test.go +++ b/src/pkg/writeback/writeback_test.go @@ -298,6 +298,91 @@ func TestBackfillGuardAndPersistence(t *testing.T) { g4.ObserveRepo("a/b") } +// A 2xx with no 403/404 before it is not the App gaining write: the repo +// was writable all along, so first_seen stays the cut-off. Only a few +// writers report outcomes, so the first one to succeed can come days after +// the hive started labelling the repo; stamping it froze every older item +// (projectbluefin/utah-packages#1148 kept a stale stage/100-merge-queue). +func TestBackfillCutoffMovesOnlyAfterARefusal(t *testing.T) { + now := t0 + deps := Deps{LevelFor: func(string) int { return 6 }, Now: func() time.Time { return now }, Logger: slog.New(slog.NewTextHandler(&bytes.Buffer{}, nil))} + labelled := Item{Number: 1148, IsPR: true, Author: "renovate[bot]", CreatedAt: t0.Add(time.Hour)} + + // 2xx only: the cut-off stays first_seen. + g := New(Config{}, deps, "") + g.ObserveRepo("org/repo") + now = t0.Add(5 * time.Hour) + g.NoteWriteOutcome("org/repo", 201) + if d := g.Allowed("org/repo", labelled, KindSentinel); !d.Allowed { + t.Fatalf("2xx with no refusal moved the cut-off: %+v", d) + } + if rs := g.Snapshot().Repos["org/repo"]; !rs.Writable || !rs.WritableAt.IsZero() || rs.PreExisting != 0 { + t.Fatalf("2xx only: %+v", rs) + } + + // 2xx, then a 403, then a 2xx: the cut-off moves to the 2xx after the refusal. + now = t0.Add(6 * time.Hour) + g.NoteWriteOutcome("org/repo", 403) + if rs := g.Snapshot().Repos["org/repo"]; rs.Writable { + t.Fatalf("an unanswered 403 still reads writable: %+v", rs) + } + now = t0.Add(7 * time.Hour) + g.NoteWriteOutcome("org/repo", 200) + if d := g.Allowed("org/repo", labelled, KindSentinel); d.Allowed || d.Reason != ReasonBackfill { + t.Fatalf("2xx after a refusal must move the cut-off: %+v", d) + } + if rs := g.Snapshot().Repos["org/repo"]; !rs.Writable || !rs.WritableAt.Equal(t0.Add(7*time.Hour)) { + t.Fatalf("after the refusal: %+v", rs) + } +} + +// A writable_at stamped by the old rule (no last_forbidden) is cleared on +// load, with the items it refused, so the repo's older items flow again; +// a record with a refusal before its writable_at is left alone. +func TestBackfillCutoffRepairOnLoad(t *testing.T) { + path := filepath.Join(t.TempDir(), "writeback.json") + state := `{"repos":{` + + `"org/stamped":{"first_seen":"2026-10-28T12:00:00Z","writable_at":"2026-10-30T08:00:00Z","pre_existing":{"1148":true,"1162":true}},` + + `"org/gained":{"first_seen":"2026-10-28T12:00:00Z","writable_at":"2026-10-30T08:00:00Z","last_forbidden":"2026-10-29T08:00:00Z","pre_existing":{"7":true}}}}` + if err := os.WriteFile(path, []byte(state), 0o644); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + deps := Deps{LevelFor: func(string) int { return 6 }, Now: func() time.Time { return t0.Add(72 * time.Hour) }, Logger: slog.New(slog.NewTextHandler(&buf, nil))} + g := New(Config{}, deps, path) + between := Item{Number: 1162, IsPR: true, Author: "h", CreatedAt: t0.Add(24 * time.Hour)} // after first_seen, before the stamped writable_at + if d := g.Allowed("org/stamped", between, KindSentinel); !d.Allowed { + t.Fatalf("repaired repo still refuses an item after first_seen: %+v", d) + } + if d := g.Allowed("org/gained", between, KindSentinel); d.Allowed || d.Reason != ReasonBackfill { + t.Fatalf("a real gain of write must keep its cut-off: %+v", d) + } + st := g.Snapshot() + if rs := st.Repos["org/stamped"]; !rs.Writable || !rs.WritableAt.IsZero() || rs.PreExisting != 0 { + t.Fatalf("stamped repo after repair: %+v", rs) + } + if rs := st.Repos["org/gained"]; !rs.Writable || rs.WritableAt.IsZero() || rs.PreExisting != 2 { + t.Fatalf("gained repo: %+v", rs) + } + for _, b := range st.Banners { + if b.Repo == "org/stamped" { + t.Fatalf("repaired repo still shows the backfill banner: %+v", b) + } + } + if !strings.Contains(buf.String(), "writeback: cleared writable_at 2026-10-30T08:00:00Z on org/stamped") || strings.Contains(buf.String(), "on org/gained") { + t.Fatalf("repair log:\n%s", buf.String()) + } + // The repair is persisted: a second load has nothing to repair. + buf.Reset() + g2 := New(Config{}, deps, path) + if rs := g2.Snapshot().Repos["org/stamped"]; !rs.WritableAt.IsZero() { + t.Fatalf("repair not persisted: %+v", rs) + } + if strings.Contains(buf.String(), "cleared writable_at") { + t.Fatalf("second load repaired again:\n%s", buf.String()) + } +} + func TestRecentAndSnapshot(t *testing.T) { now := t0 g := New(Config{}, Deps{LevelFor: func(string) int { return 4 }, OptInLabels: func(string) []string { return []string{"needs-hive"} }, Now: func() time.Time { return now }}, "")