From 4d12ffb087aadccfdc8afca73d4ea3e071966a57 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 28 Oct 2026 00:10:32 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20convergence:=20bound=20the=20mut?= =?UTF-8?q?ation=20journal=20so=20memory=20stops=20growing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The operation journal (journal.json) kept every Applied/NotApplied entry forever. On the hosted hive it reached 53k entries / 56 MB and, because every Begin/RecordResult/Reconcile reloads the file, clones every entry and rewrites the whole snapshot with MarshalIndent, it accounted for ~60% of all Go allocations (~13 GB/min) and a persistent ~44 MB heap residue. Terminal entries now age out after DefaultJournalRetention (7 days) and the oldest terminal entries are evicted past DefaultJournalMaxEntries (20k). Planned/Unknown entries are never pruned: they still need reconciliation. Logical IDs embed the desired generation and claim key, so a week-old terminal entry can no longer be replayed by a live planner. Signed-off-by: Andy Anderson --- .../fixed-mutation-journal-retention.md | 1 + src/pkg/convergence/mutation/journal.go | 73 +++++++++- .../mutation/journal_prune_test.go | 128 ++++++++++++++++++ 3 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-mutation-journal-retention.md create mode 100644 src/pkg/convergence/mutation/journal_prune_test.go diff --git a/changelog.d/fixed-mutation-journal-retention.md b/changelog.d/fixed-mutation-journal-retention.md new file mode 100644 index 0000000000..86f9364ae7 --- /dev/null +++ b/changelog.d/fixed-mutation-journal-retention.md @@ -0,0 +1 @@ +- Bound the convergence mutation journal: terminal (Applied/NotApplied) operations age out after 7 days and the oldest terminal entries are evicted past 20k entries, so `journal.json` — reloaded and fully rewritten on every label/comment/merge transition — stops growing without limit (53k entries / 56 MB on the hosted hive, ~60% of all Go allocations). Planned/Unknown entries are never pruned. diff --git a/src/pkg/convergence/mutation/journal.go b/src/pkg/convergence/mutation/journal.go index 63c49e05bd..e82f57e83e 100644 --- a/src/pkg/convergence/mutation/journal.go +++ b/src/pkg/convergence/mutation/journal.go @@ -94,6 +94,20 @@ const journalLockFileSuffix = ".lock" // cross-process lock before surfacing ErrJournalLocked. const DefaultJournalLockTimeout = 10 * time.Second +// DefaultJournalRetention bounds how long a TERMINAL operation (Applied or +// NotApplied) stays in the journal after its last transition. Every logical +// ID embeds the outcome's desired generation and the authorizing claim key, +// so a terminal entry this old can no longer be replayed by a live planner; +// keeping it only grows the file every transition rewrites and reloads. +// Planned and Unknown entries are never pruned: they still need +// reconciliation against authoritative external state. +const DefaultJournalRetention = 7 * 24 * time.Hour + +// DefaultJournalMaxEntries is the hard ceiling on journal entries. Past it the +// oldest terminal entries are evicted first (never Planned/Unknown), so a +// burst of transitions cannot outgrow the retention window's bound either. +const DefaultJournalMaxEntries = 20000 + // journalLockPollInterval is the spacing between non-blocking flock attempts // while waiting out the bounded lock timeout. const journalLockPollInterval = 10 * time.Millisecond @@ -209,10 +223,14 @@ type persistedJournal struct { // transition, serialized per logical-ID under the journal mutex and across // processes under the exclusive .lock flock (refreshed from disk before every // transition, exactly like the claim ledger). Process memory is never -// authority. +// authority. Because every transition reloads and rewrites the whole file, +// the record is bounded: terminal entries age out after retention and the +// oldest terminal entries are evicted past maxEntries (see pruneLocked). type Journal struct { path string lockTimeout time.Duration + retention time.Duration + maxEntries int mu sync.Mutex ops map[string]*Operation } @@ -224,7 +242,13 @@ type Journal struct { // transition, exactly like the claim ledger, so a missing or unwritable data // directory defers persistence rather than refusing boot. func OpenJournal(path string) (*Journal, error) { - j := &Journal{path: path, lockTimeout: DefaultJournalLockTimeout, ops: make(map[string]*Operation)} + j := &Journal{ + path: path, + lockTimeout: DefaultJournalLockTimeout, + retention: DefaultJournalRetention, + maxEntries: DefaultJournalMaxEntries, + ops: make(map[string]*Operation), + } if err := j.reloadLocked(); err != nil { return nil, err } @@ -502,6 +526,7 @@ func (j *Journal) commit(id string, work Operation) (Operation, error) { prior, hadPrior := j.ops[id] cp := work.clone() j.ops[id] = &cp + j.pruneLocked(work.UpdatedAt) if err := j.persistLocked(); err != nil { if hadPrior { j.ops[id] = prior @@ -513,6 +538,50 @@ func (j *Journal) commit(id string, work Operation) (Operation, error) { return work.clone(), nil } +// isTerminal reports whether an operation can never transition again on its +// own: Applied is terminal truth and NotApplied only moves through a fresh +// Begin, which re-derives the same logical ID from the effect. +func isTerminal(status string) bool { + return status == StatusApplied || status == StatusNotApplied +} + +// pruneLocked bounds the in-memory index (and so the next persisted +// snapshot) before a commit: terminal entries past the retention window are +// dropped, then the oldest terminal entries are evicted until the count fits +// maxEntries. Planned and Unknown entries are always kept. Callers hold j.mu +// and the cross-process lock. +func (j *Journal) pruneLocked(now time.Time) { + if j.retention > 0 { + cutoff := now.Add(-j.retention) + for id, op := range j.ops { + if isTerminal(op.Status) && op.UpdatedAt.Before(cutoff) { + delete(j.ops, id) + } + } + } + if j.maxEntries <= 0 || len(j.ops) <= j.maxEntries { + return + } + terminal := make([]*Operation, 0, len(j.ops)) + for _, op := range j.ops { + if isTerminal(op.Status) { + terminal = append(terminal, op) + } + } + sort.Slice(terminal, func(i, k int) bool { + if !terminal[i].UpdatedAt.Equal(terminal[k].UpdatedAt) { + return terminal[i].UpdatedAt.Before(terminal[k].UpdatedAt) + } + return terminal[i].LogicalID < terminal[k].LogicalID + }) + for _, op := range terminal { + if len(j.ops) <= j.maxEntries { + break + } + delete(j.ops, op.LogicalID) + } +} + func (j *Journal) persistLocked() error { all := make([]Operation, 0, len(j.ops)) for _, op := range j.ops { diff --git a/src/pkg/convergence/mutation/journal_prune_test.go b/src/pkg/convergence/mutation/journal_prune_test.go new file mode 100644 index 0000000000..b6975c0a7f --- /dev/null +++ b/src/pkg/convergence/mutation/journal_prune_test.go @@ -0,0 +1,128 @@ +package mutation + +import ( + "fmt" + "path/filepath" + "testing" + "time" +) + +// effectN derives a distinct logical operation per generation so a test can +// populate the journal with many independent entries. +func effectN(n int) Effect { + e := testEffect() + e.DesiredGeneration = n + e.Inputs = map[string]string{"repo": "acme/widgets", "head": fmt.Sprintf("feature-%d", n), "base": "main"} + return e +} + +// applyN journals one Applied operation for generation n at instant at. +func applyN(t *testing.T, j *Journal, n int, at time.Time) Operation { + t.Helper() + op, err := j.Begin(effectN(n), 1, "alice", at) + if err != nil { + t.Fatalf("Begin %d: %v", n, err) + } + op, err = j.RecordResult(op.LogicalID, 1, StatusApplied, fmt.Sprintf("pr-%d", n), at) + if err != nil { + t.Fatalf("RecordResult %d: %v", n, err) + } + return op +} + +func countOps(j *Journal) int { + j.mu.Lock() + defer j.mu.Unlock() + return len(j.ops) +} + +// A terminal entry older than the retention window is dropped by the next +// commit, both in memory and from the persisted snapshot; a fresh terminal +// entry and an unresolved entry survive. +func TestJournal_PrunesTerminalEntriesPastRetention(t *testing.T) { + path := filepath.Join(t.TempDir(), "journal.json") + j, err := OpenJournal(path) + if err != nil { + t.Fatalf("OpenJournal: %v", err) + } + j.retention = time.Hour + base := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) + + stale := applyN(t, j, 1, base) + planned, err := j.Begin(effectN(2), 1, "alice", base) + if err != nil { + t.Fatalf("Begin planned: %v", err) + } + fresh := applyN(t, j, 3, base.Add(2*time.Hour)) + + if _, ok := j.Get(stale.LogicalID); ok { + t.Fatal("terminal entry past retention must be pruned") + } + if _, ok := j.Get(planned.LogicalID); !ok { + t.Fatal("unresolved (Planned) entry must never be pruned regardless of age") + } + if _, ok := j.Get(fresh.LogicalID); !ok { + t.Fatal("terminal entry inside the retention window must be kept") + } + + reopened, err := OpenJournal(path) + if err != nil { + t.Fatalf("reopen: %v", err) + } + if got := countOps(reopened); got != 2 { + t.Fatalf("persisted snapshot must reflect the prune: %d entries, want 2", got) + } +} + +// The entry count never exceeds maxEntries: the oldest terminal entries are +// evicted first while Planned/Unknown entries are always retained. +func TestJournal_EvictsOldestTerminalEntriesPastMaxEntries(t *testing.T) { + j := openJournal(t) + j.retention = 0 + j.maxEntries = 5 + base := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) + + unknown, err := j.Begin(effectN(100), 1, "alice", base) + if err != nil { + t.Fatalf("Begin unknown: %v", err) + } + if _, err := j.Reconcile(unknown.LogicalID, ExternalState{Known: false}, base); err != nil { + t.Fatalf("Reconcile unknown: %v", err) + } + + var ids []string + for n := 1; n <= 12; n++ { + op := applyN(t, j, n, base.Add(time.Duration(n)*time.Minute)) + ids = append(ids, op.LogicalID) + if got := countOps(j); got > j.maxEntries { + t.Fatalf("after %d commits journal holds %d entries, bound is %d", n, got, j.maxEntries) + } + } + if got := countOps(j); got != j.maxEntries { + t.Fatalf("journal should sit exactly at the bound: %d != %d", got, j.maxEntries) + } + if _, ok := j.Get(unknown.LogicalID); !ok { + t.Fatal("Unknown entry must survive eviction") + } + // Four newest terminal entries remain alongside the Unknown one. + for i, id := range ids { + _, ok := j.Get(id) + if wantKept := i >= len(ids)-(j.maxEntries-1); ok != wantKept { + t.Fatalf("entry %d kept=%v want %v", i+1, ok, wantKept) + } + } +} + +// With bounds disabled (zero values) nothing is ever pruned. +func TestJournal_ZeroBoundsDisablePruning(t *testing.T) { + j := openJournal(t) + j.retention = 0 + j.maxEntries = 0 + base := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) + for n := 1; n <= 5; n++ { + applyN(t, j, n, base.Add(time.Duration(n)*365*24*time.Hour)) + } + if got := countOps(j); got != 5 { + t.Fatalf("unbounded journal must keep every entry: %d", got) + } +}