Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-writeback-cutoff-needs-prior-refusal.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Write-back backfill guard: a successful write with no 403/404 before it no longer moves a repository's backfill cut-off (`writable_at`). Only a few writers report outcomes, so the first one to succeed could come days after onboarding, and the guard then froze the stage labels of every older PR and issue the hive had been labelling all along. A `writable_at` stamped that way is cleared at startup (logged as `writeback: cleared writable_at …`) with the items it refused, and `/api/status` `writeback.repos[].writable` now reads true for a repository that never refused a write.
14 changes: 9 additions & 5 deletions src/docs/operator-reference/sweeps-and-operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -671,11 +671,15 @@ and the backfill banner. `/api/status` carries the same state under

The gate persists `first_seen` per repo in `/data/metrics/writeback.json`
(`HIVE_WRITEBACK_STATE_PATH`), and records the moment a repo first answers
2xx after 403/404s as `writable_at`. Items created before the later of the
two are never written to unsolicited while `backfill: false`. The sweep logs
one line — `writeback: skipped N pre-existing items on <repo> (backfill
off)` — and the dashboard shows "hive can now write to `<repo>`; N existing
PRs were left untouched — enable backfill to process them".
2xx after 403/404s as `writable_at`. A 2xx with no 403/404 before it does
not set `writable_at`: the repo was writable all along, so `first_seen` alone
is the cut-off (a `writable_at` an older hive stamped that way is cleared at
startup, logged as `writeback: cleared writable_at …`). Items created before
the later of the two are never written to unsolicited while
`backfill: false`. The sweep logs one line — `writeback: skipped N
pre-existing items on <repo> (backfill off)` — and the dashboard shows
"hive can now write to `<repo>`; N existing PRs were left untouched —
enable backfill to process them".

### Duplicate-write suppression

Expand Down
40 changes: 35 additions & 5 deletions src/pkg/writeback/gate.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,10 @@ type Suppressed struct {

// RepoStatus is one repo's block in the status payload.
type RepoStatus struct {
FirstSeen time.Time `json:"first_seen,omitempty"`
FirstSeen time.Time `json:"first_seen,omitempty"`
// Writable is false only while a 403/404 has had no 2xx after it; a
// repo that never refused a write was writable all along and carries
// no WritableAt.
Writable bool `json:"writable"`
WritableAt time.Time `json:"writable_at,omitempty"`
Mode RepoMode `json:"mode,omitempty"`
Expand Down Expand Up @@ -188,7 +191,8 @@ type repoRecord struct {
FirstSeen time.Time `json:"first_seen"`
WritableAt time.Time `json:"writable_at,omitempty"`
// LastForbidden is the last 403/404 a write to this repo got, so the
// first 2xx after it marks the moment the App gained write.
// first 2xx after it marks the moment the App gained write. A repo that
// never got one was writable all along and keeps WritableAt zero.
LastForbidden time.Time `json:"last_forbidden,omitempty"`
// PreExisting is the set of item numbers the backfill guard refused.
PreExisting map[int]bool `json:"pre_existing,omitempty"`
Expand Down Expand Up @@ -269,9 +273,32 @@ func New(cfg Config, deps Deps, path string) *Gate {
g.deps.Logger = slog.Default()
}
g.load()
g.repairBackfillCutoffs()
return g
}

// repairBackfillCutoffs clears a writable_at that NoteWriteOutcome stamped
// on a 2xx with no 403/404 before it, before it required one: that repo was
// writable all along, so first_seen alone is the cut-off. The items the
// wrong cut-off refused are dropped too; the guard records again any that
// really predate first_seen the next time it refuses one.
func (g *Gate) repairBackfillCutoffs() {
repaired := false
for k, rec := range g.repos {
if rec == nil || rec.WritableAt.IsZero() || !rec.LastForbidden.IsZero() {
continue
}
g.deps.Logger.Info(fmt.Sprintf("writeback: cleared writable_at %s on %s (no 403/404 before it); backfill cut-off is first_seen %s",
rec.WritableAt.Format(time.RFC3339), k, rec.FirstSeen.Format(time.RFC3339)))
rec.WritableAt = time.Time{}
rec.PreExisting = nil
repaired = true
}
if repaired {
g.saveLocked()
}
}

// SetConfig swaps the policy (dashboard edits take effect on the next
// decision).
func (g *Gate) SetConfig(cfg Config) {
Expand Down Expand Up @@ -384,7 +411,10 @@ func (g *Gate) ObserveRepo(repo string) {
// NoteWriteOutcome records the result of a real GitHub write to repo so the
// gate can detect the App gaining write: a 403/404 marks the repo as not
// writable; the first 2xx after one marks WritableAt now, which the
// backfill guard uses as the cut-off for items created before it.
// backfill guard uses as the cut-off for items created before it. A 2xx
// with no 403/404 before it is not a gain: only a few writers report here,
// so the first one to succeed can come days after the hive started
// labelling the repo, and stamping it froze every older item.
func (g *Gate) NoteWriteOutcome(repo string, statusCode int) {
if g == nil || strings.TrimSpace(repo) == "" {
return
Expand All @@ -398,7 +428,7 @@ func (g *Gate) NoteWriteOutcome(repo string, statusCode int) {
rec.LastForbidden = now
g.saveLocked()
case statusCode >= 200 && statusCode < 300:
if rec.WritableAt.IsZero() {
if rec.WritableAt.IsZero() && !rec.LastForbidden.IsZero() {
rec.WritableAt = now
g.saveLocked()
}
Expand Down Expand Up @@ -763,7 +793,7 @@ func (g *Gate) Snapshot() Status {
if g.deps.LevelFor != nil {
level = g.deps.LevelFor(k)
}
rs := RepoStatus{FirstSeen: rec.FirstSeen, Writable: !rec.WritableAt.IsZero(), WritableAt: rec.WritableAt, Mode: cfg.RepoModeFor(k), Level: level, Suppressed24: per[k], PreExisting: len(rec.PreExisting)}
rs := RepoStatus{FirstSeen: rec.FirstSeen, Writable: !rec.WritableAt.IsZero() || rec.LastForbidden.IsZero(), WritableAt: rec.WritableAt, Mode: cfg.RepoModeFor(k), Level: level, Suppressed24: per[k], PreExisting: len(rec.PreExisting)}
st.Repos[k] = rs
if rs.PreExisting > 0 && !cfg.BackfillEnabled() {
st.Banners = append(st.Banners, Banner{Repo: k, WritableAt: rec.WritableAt, Untouched: rs.PreExisting,
Expand Down
85 changes: 85 additions & 0 deletions src/pkg/writeback/writeback_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,91 @@ func TestBackfillGuardAndPersistence(t *testing.T) {
g4.ObserveRepo("a/b")
}

// A 2xx with no 403/404 before it is not the App gaining write: the repo
// was writable all along, so first_seen stays the cut-off. Only a few
// writers report outcomes, so the first one to succeed can come days after
// the hive started labelling the repo; stamping it froze every older item
// (projectbluefin/utah-packages#1148 kept a stale stage/100-merge-queue).
func TestBackfillCutoffMovesOnlyAfterARefusal(t *testing.T) {
now := t0
deps := Deps{LevelFor: func(string) int { return 6 }, Now: func() time.Time { return now }, Logger: slog.New(slog.NewTextHandler(&bytes.Buffer{}, nil))}
labelled := Item{Number: 1148, IsPR: true, Author: "renovate[bot]", CreatedAt: t0.Add(time.Hour)}

// 2xx only: the cut-off stays first_seen.
g := New(Config{}, deps, "")
g.ObserveRepo("org/repo")
now = t0.Add(5 * time.Hour)
g.NoteWriteOutcome("org/repo", 201)
if d := g.Allowed("org/repo", labelled, KindSentinel); !d.Allowed {
t.Fatalf("2xx with no refusal moved the cut-off: %+v", d)
}
if rs := g.Snapshot().Repos["org/repo"]; !rs.Writable || !rs.WritableAt.IsZero() || rs.PreExisting != 0 {
t.Fatalf("2xx only: %+v", rs)
}

// 2xx, then a 403, then a 2xx: the cut-off moves to the 2xx after the refusal.
now = t0.Add(6 * time.Hour)
g.NoteWriteOutcome("org/repo", 403)
if rs := g.Snapshot().Repos["org/repo"]; rs.Writable {
t.Fatalf("an unanswered 403 still reads writable: %+v", rs)
}
now = t0.Add(7 * time.Hour)
g.NoteWriteOutcome("org/repo", 200)
if d := g.Allowed("org/repo", labelled, KindSentinel); d.Allowed || d.Reason != ReasonBackfill {
t.Fatalf("2xx after a refusal must move the cut-off: %+v", d)
}
if rs := g.Snapshot().Repos["org/repo"]; !rs.Writable || !rs.WritableAt.Equal(t0.Add(7*time.Hour)) {
t.Fatalf("after the refusal: %+v", rs)
}
}

// A writable_at stamped by the old rule (no last_forbidden) is cleared on
// load, with the items it refused, so the repo's older items flow again;
// a record with a refusal before its writable_at is left alone.
func TestBackfillCutoffRepairOnLoad(t *testing.T) {
path := filepath.Join(t.TempDir(), "writeback.json")
state := `{"repos":{` +
`"org/stamped":{"first_seen":"2026-10-28T12:00:00Z","writable_at":"2026-10-30T08:00:00Z","pre_existing":{"1148":true,"1162":true}},` +
`"org/gained":{"first_seen":"2026-10-28T12:00:00Z","writable_at":"2026-10-30T08:00:00Z","last_forbidden":"2026-10-29T08:00:00Z","pre_existing":{"7":true}}}}`
if err := os.WriteFile(path, []byte(state), 0o644); err != nil {
t.Fatal(err)
}
var buf bytes.Buffer
deps := Deps{LevelFor: func(string) int { return 6 }, Now: func() time.Time { return t0.Add(72 * time.Hour) }, Logger: slog.New(slog.NewTextHandler(&buf, nil))}
g := New(Config{}, deps, path)
between := Item{Number: 1162, IsPR: true, Author: "h", CreatedAt: t0.Add(24 * time.Hour)} // after first_seen, before the stamped writable_at
if d := g.Allowed("org/stamped", between, KindSentinel); !d.Allowed {
t.Fatalf("repaired repo still refuses an item after first_seen: %+v", d)
}
if d := g.Allowed("org/gained", between, KindSentinel); d.Allowed || d.Reason != ReasonBackfill {
t.Fatalf("a real gain of write must keep its cut-off: %+v", d)
}
st := g.Snapshot()
if rs := st.Repos["org/stamped"]; !rs.Writable || !rs.WritableAt.IsZero() || rs.PreExisting != 0 {
t.Fatalf("stamped repo after repair: %+v", rs)
}
if rs := st.Repos["org/gained"]; !rs.Writable || rs.WritableAt.IsZero() || rs.PreExisting != 2 {
t.Fatalf("gained repo: %+v", rs)
}
for _, b := range st.Banners {
if b.Repo == "org/stamped" {
t.Fatalf("repaired repo still shows the backfill banner: %+v", b)
}
}
if !strings.Contains(buf.String(), "writeback: cleared writable_at 2026-10-30T08:00:00Z on org/stamped") || strings.Contains(buf.String(), "on org/gained") {
t.Fatalf("repair log:\n%s", buf.String())
}
// The repair is persisted: a second load has nothing to repair.
buf.Reset()
g2 := New(Config{}, deps, path)
if rs := g2.Snapshot().Repos["org/stamped"]; !rs.WritableAt.IsZero() {
t.Fatalf("repair not persisted: %+v", rs)
}
if strings.Contains(buf.String(), "cleared writable_at") {
t.Fatalf("second load repaired again:\n%s", buf.String())
}
}

func TestRecentAndSnapshot(t *testing.T) {
now := t0
g := New(Config{}, Deps{LevelFor: func(string) int { return 4 }, OptInLabels: func(string) []string { return []string{"needs-hive"} }, Now: func() time.Time { return now }}, "")
Expand Down
Loading