Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-mutation-journal-retention.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Bound the convergence mutation journal: terminal (Applied/NotApplied) operations age out after 7 days and the oldest terminal entries are evicted past 20k entries, so `journal.json` — reloaded and fully rewritten on every label/comment/merge transition — stops growing without limit (53k entries / 56 MB on the hosted hive, ~60% of all Go allocations). Planned/Unknown entries are never pruned.
73 changes: 71 additions & 2 deletions src/pkg/convergence/mutation/journal.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package mutation

import (
Expand Down Expand Up @@ -94,6 +94,20 @@
// cross-process lock before surfacing ErrJournalLocked.
const DefaultJournalLockTimeout = 10 * time.Second

// DefaultJournalRetention bounds how long a TERMINAL operation (Applied or
// NotApplied) stays in the journal after its last transition. Every logical
// ID embeds the outcome's desired generation and the authorizing claim key,
// so a terminal entry this old can no longer be replayed by a live planner;
// keeping it only grows the file every transition rewrites and reloads.
// Planned and Unknown entries are never pruned: they still need
// reconciliation against authoritative external state.
const DefaultJournalRetention = 7 * 24 * time.Hour

// DefaultJournalMaxEntries is the hard ceiling on journal entries. Past it the
// oldest terminal entries are evicted first (never Planned/Unknown), so a
// burst of transitions cannot outgrow the retention window's bound either.
const DefaultJournalMaxEntries = 20000

// journalLockPollInterval is the spacing between non-blocking flock attempts
// while waiting out the bounded lock timeout.
const journalLockPollInterval = 10 * time.Millisecond
Expand Down Expand Up @@ -209,10 +223,14 @@
// transition, serialized per logical-ID under the journal mutex and across
// processes under the exclusive .lock flock (refreshed from disk before every
// transition, exactly like the claim ledger). Process memory is never
// authority.
// authority. Because every transition reloads and rewrites the whole file,
// the record is bounded: terminal entries age out after retention and the
// oldest terminal entries are evicted past maxEntries (see pruneLocked).
type Journal struct {
path string
lockTimeout time.Duration
retention time.Duration
maxEntries int
mu sync.Mutex
ops map[string]*Operation
}
Expand All @@ -224,7 +242,13 @@
// transition, exactly like the claim ledger, so a missing or unwritable data
// directory defers persistence rather than refusing boot.
func OpenJournal(path string) (*Journal, error) {
j := &Journal{path: path, lockTimeout: DefaultJournalLockTimeout, ops: make(map[string]*Operation)}
j := &Journal{
path: path,
lockTimeout: DefaultJournalLockTimeout,
retention: DefaultJournalRetention,
maxEntries: DefaultJournalMaxEntries,
ops: make(map[string]*Operation),
}
if err := j.reloadLocked(); err != nil {
return nil, err
}
Expand Down Expand Up @@ -502,6 +526,7 @@
prior, hadPrior := j.ops[id]
cp := work.clone()
j.ops[id] = &cp
j.pruneLocked(work.UpdatedAt)
if err := j.persistLocked(); err != nil {
if hadPrior {
j.ops[id] = prior
Expand All @@ -513,6 +538,50 @@
return work.clone(), nil
}

// isTerminal reports whether an operation can never transition again on its
// own: Applied is terminal truth and NotApplied only moves through a fresh
// Begin, which re-derives the same logical ID from the effect.
func isTerminal(status string) bool {
return status == StatusApplied || status == StatusNotApplied
}

// pruneLocked bounds the in-memory index (and so the next persisted
// snapshot) before a commit: terminal entries past the retention window are
// dropped, then the oldest terminal entries are evicted until the count fits
// maxEntries. Planned and Unknown entries are always kept. Callers hold j.mu
// and the cross-process lock.
func (j *Journal) pruneLocked(now time.Time) {
if j.retention > 0 {
cutoff := now.Add(-j.retention)
for id, op := range j.ops {
if isTerminal(op.Status) && op.UpdatedAt.Before(cutoff) {
delete(j.ops, id)
}
}
}
if j.maxEntries <= 0 || len(j.ops) <= j.maxEntries {
return
}
terminal := make([]*Operation, 0, len(j.ops))
for _, op := range j.ops {
if isTerminal(op.Status) {
terminal = append(terminal, op)
}
}
sort.Slice(terminal, func(i, k int) bool {
if !terminal[i].UpdatedAt.Equal(terminal[k].UpdatedAt) {
return terminal[i].UpdatedAt.Before(terminal[k].UpdatedAt)
}
return terminal[i].LogicalID < terminal[k].LogicalID
})
for _, op := range terminal {
if len(j.ops) <= j.maxEntries {
break
}
delete(j.ops, op.LogicalID)
}
}

func (j *Journal) persistLocked() error {
all := make([]Operation, 0, len(j.ops))
for _, op := range j.ops {
Expand Down
128 changes: 128 additions & 0 deletions src/pkg/convergence/mutation/journal_prune_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
package mutation

import (
"fmt"
"path/filepath"
"testing"
"time"
)

// effectN derives a distinct logical operation per generation so a test can
// populate the journal with many independent entries.
func effectN(n int) Effect {
e := testEffect()
e.DesiredGeneration = n
e.Inputs = map[string]string{"repo": "acme/widgets", "head": fmt.Sprintf("feature-%d", n), "base": "main"}
return e
}

// applyN journals one Applied operation for generation n at instant at.
func applyN(t *testing.T, j *Journal, n int, at time.Time) Operation {
t.Helper()
op, err := j.Begin(effectN(n), 1, "alice", at)
if err != nil {
t.Fatalf("Begin %d: %v", n, err)
}
op, err = j.RecordResult(op.LogicalID, 1, StatusApplied, fmt.Sprintf("pr-%d", n), at)
if err != nil {
t.Fatalf("RecordResult %d: %v", n, err)
}
return op
}

func countOps(j *Journal) int {
j.mu.Lock()
defer j.mu.Unlock()
return len(j.ops)
}

// A terminal entry older than the retention window is dropped by the next
// commit, both in memory and from the persisted snapshot; a fresh terminal
// entry and an unresolved entry survive.
func TestJournal_PrunesTerminalEntriesPastRetention(t *testing.T) {
path := filepath.Join(t.TempDir(), "journal.json")
j, err := OpenJournal(path)
if err != nil {
t.Fatalf("OpenJournal: %v", err)
}
j.retention = time.Hour
base := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)

stale := applyN(t, j, 1, base)
planned, err := j.Begin(effectN(2), 1, "alice", base)
if err != nil {
t.Fatalf("Begin planned: %v", err)
}
fresh := applyN(t, j, 3, base.Add(2*time.Hour))

if _, ok := j.Get(stale.LogicalID); ok {
t.Fatal("terminal entry past retention must be pruned")
}
if _, ok := j.Get(planned.LogicalID); !ok {
t.Fatal("unresolved (Planned) entry must never be pruned regardless of age")
}
if _, ok := j.Get(fresh.LogicalID); !ok {
t.Fatal("terminal entry inside the retention window must be kept")
}

reopened, err := OpenJournal(path)
if err != nil {
t.Fatalf("reopen: %v", err)
}
if got := countOps(reopened); got != 2 {
t.Fatalf("persisted snapshot must reflect the prune: %d entries, want 2", got)
}
}

// The entry count never exceeds maxEntries: the oldest terminal entries are
// evicted first while Planned/Unknown entries are always retained.
func TestJournal_EvictsOldestTerminalEntriesPastMaxEntries(t *testing.T) {
j := openJournal(t)
j.retention = 0
j.maxEntries = 5
base := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)

unknown, err := j.Begin(effectN(100), 1, "alice", base)
if err != nil {
t.Fatalf("Begin unknown: %v", err)
}
if _, err := j.Reconcile(unknown.LogicalID, ExternalState{Known: false}, base); err != nil {
t.Fatalf("Reconcile unknown: %v", err)
}

var ids []string
for n := 1; n <= 12; n++ {
op := applyN(t, j, n, base.Add(time.Duration(n)*time.Minute))
ids = append(ids, op.LogicalID)
if got := countOps(j); got > j.maxEntries {
t.Fatalf("after %d commits journal holds %d entries, bound is %d", n, got, j.maxEntries)
}
}
if got := countOps(j); got != j.maxEntries {
t.Fatalf("journal should sit exactly at the bound: %d != %d", got, j.maxEntries)
}
if _, ok := j.Get(unknown.LogicalID); !ok {
t.Fatal("Unknown entry must survive eviction")
}
// Four newest terminal entries remain alongside the Unknown one.
for i, id := range ids {
_, ok := j.Get(id)
if wantKept := i >= len(ids)-(j.maxEntries-1); ok != wantKept {
t.Fatalf("entry %d kept=%v want %v", i+1, ok, wantKept)
}
}
}

// With bounds disabled (zero values) nothing is ever pruned.
func TestJournal_ZeroBoundsDisablePruning(t *testing.T) {
j := openJournal(t)
j.retention = 0
j.maxEntries = 0
base := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)
for n := 1; n <= 5; n++ {
applyN(t, j, n, base.Add(time.Duration(n)*365*24*time.Hour))
}
if got := countOps(j); got != 5 {
t.Fatalf("unbounded journal must keep every entry: %d", got)
}
}
Loading