From 847e32407a1b066c233ae166d4914a106474e38c Mon Sep 17 00:00:00 2001 From: protonmatter Date: Tue, 17 Nov 2026 13:04:36 -0500 Subject: [PATCH 1/4] Coalesce circuit-breaker writes onto one queue. A newer state for the same profile replaces one that has not been written yet, and a failed write is logged instead of discarded on a detached goroutine. --- server/llmpool.go | 102 ++++++++++++++++++++++++++++++---- server/llmpool_health_test.go | 51 +++++++++++++++++ 2 files changed, 142 insertions(+), 11 deletions(-) create mode 100644 server/llmpool_health_test.go diff --git a/server/llmpool.go b/server/llmpool.go index a3901fcfa..65b946522 100644 --- a/server/llmpool.go +++ b/server/llmpool.go @@ -2,6 +2,7 @@ package server import ( "log" + "sync" "time" "github.com/Autumn-27/artex/agent" @@ -23,20 +24,12 @@ func newLLMHealthRegistry(pg *db.DB) *llmpool.Registry { if pg == nil { return llmpool.NewRegistry(nil, nil) } + syncer := newHealthSync(pg.SaveLLMHealth, pg.ClearLLMHealth) persist := func(id int64, st llmpool.State) { - h := db.LLMHealth{ProfileID: id, Fails: st.Fails, Trips: st.Trips, LastError: st.LastError} - if !st.OpenUntil.IsZero() { - t := st.OpenUntil - h.OpenUntil = &t - } - go func() { - if err := pg.SaveLLMHealth(h); err != nil { - log.Printf("[llmpool] failed to persist circuit-breaker state: %v", err) - } - }() + syncer.enqueue(id, healthOp{st: st}) } forget := func(id int64) { - go func() { _ = pg.ClearLLMHealth(id) }() + syncer.enqueue(id, healthOp{clear: true}) } reg := llmpool.NewRegistry(persist, forget) // Restore only windows that haven't expired (LoadLLMHealth filters), so a @@ -54,6 +47,93 @@ func newLLMHealthRegistry(pg *db.DB) *llmpool.Registry { return reg } +// healthOp is the latest circuit-breaker write for one profile. clear drops the row. +type healthOp struct { + clear bool + st llmpool.State +} + +// healthSync writes circuit-breaker state from one goroutine. A newer update for +// the same profile replaces one that has not been written yet, so a burst of +// state changes becomes one database call and the request path does not wait. +type healthSync struct { + save func(db.LLMHealth) error + clear func(int64) error + mu sync.Mutex + pending map[int64]healthOp + wake chan struct{} +} + +func newHealthSync(save func(db.LLMHealth) error, clear func(int64) error) *healthSync { + h := &healthSync{ + save: save, + clear: clear, + pending: map[int64]healthOp{}, + wake: make(chan struct{}, 1), + } + go h.loop() + return h +} + +func (h *healthSync) enqueue(id int64, op healthOp) { + h.mu.Lock() + if h.pending == nil { + h.pending = map[int64]healthOp{} + } + h.pending[id] = op + h.mu.Unlock() + if h.wake == nil { + return + } + select { + case h.wake <- struct{}{}: + default: + } +} + +func (h *healthSync) take() map[int64]healthOp { + h.mu.Lock() + defer h.mu.Unlock() + if len(h.pending) == 0 { + return nil + } + out := h.pending + h.pending = map[int64]healthOp{} + return out +} + +func (h *healthSync) loop() { + for { + batch := h.take() + if len(batch) == 0 { + <-h.wake + continue + } + for id, op := range batch { + if op.clear { + if h.clear == nil { + continue + } + if err := h.clear(id); err != nil { + log.Printf("[llmpool] clear circuit-breaker profile #%d: %v", id, err) + } + continue + } + if h.save == nil { + continue + } + row := db.LLMHealth{ProfileID: id, Fails: op.st.Fails, Trips: op.st.Trips, LastError: op.st.LastError} + if !op.st.OpenUntil.IsZero() { + openUntil := op.st.OpenUntil + row.OpenUntil = &openUntil + } + if err := h.save(row); err != nil { + log.Printf("[llmpool] persist circuit-breaker profile #%d: %v", id, err) + } + } + } +} + // poolMember builds one chain member from a profile, reusing the per-profile // provider cache so every agent pointing at the same profile shares one provider // (hence one rate limiter). Returns nil when the profile can't be built. diff --git a/server/llmpool_health_test.go b/server/llmpool_health_test.go new file mode 100644 index 000000000..c759d2c08 --- /dev/null +++ b/server/llmpool_health_test.go @@ -0,0 +1,51 @@ +package server + +import ( + "testing" + + "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/artex/llmpool" +) + +func TestHealthSyncCoalescesToLatest(t *testing.T) { + h := &healthSync{pending: map[int64]healthOp{}} + h.enqueue(7, healthOp{st: llmpool.State{Fails: 1}}) + h.enqueue(7, healthOp{st: llmpool.State{Fails: 4}}) + h.enqueue(7, healthOp{clear: true}) + h.enqueue(8, healthOp{st: llmpool.State{Fails: 2}}) + batch := h.take() + if len(batch) != 2 || !batch[7].clear || batch[8].st.Fails != 2 { + t.Fatalf("batch=%+v", batch) + } + if again := h.take(); again != nil { + t.Fatalf("pending was not cleared: %+v", again) + } + var saved []db.LLMHealth + var cleared []int64 + worker := &healthSync{ + save: func(row db.LLMHealth) error { + saved = append(saved, row) + return nil + }, + clear: func(id int64) error { + cleared = append(cleared, id) + return nil + }, + } + worker.enqueue(7, healthOp{clear: true}) + worker.enqueue(8, healthOp{st: llmpool.State{Fails: 3, LastError: "down"}}) + for id, op := range worker.take() { + if op.clear { + if err := worker.clear(id); err != nil { + t.Fatal(err) + } + continue + } + if err := worker.save(db.LLMHealth{ProfileID: id, Fails: op.st.Fails, LastError: op.st.LastError}); err != nil { + t.Fatal(err) + } + } + if len(cleared) != 1 || cleared[0] != 7 || len(saved) != 1 || saved[0].ProfileID != 8 || saved[0].Fails != 3 { + t.Fatalf("cleared=%v saved=%+v", cleared, saved) + } +} From 1fbf2263a120aebf79ee55948c95d328f9d1e9b1 Mon Sep 17 00:00:00 2001 From: protonmatter Date: Tue, 17 Nov 2026 15:55:13 -0500 Subject: [PATCH 2/4] Keep circuit-breaker writes in state order and flush them before the database closes. A later update can no longer be queued behind an older one, a failed save is retried, and shutdown waits for the last cooling window. --- cmd/artex/main.go | 3 + llmpool/health.go | 29 +++--- llmpool/health_order_test.go | 51 ++++++++++ server/llmpool.go | 180 +++++++++++++++++++++++++--------- server/llmpool_health_test.go | 125 ++++++++++++++++------- server/server.go | 7 +- 6 files changed, 299 insertions(+), 96 deletions(-) create mode 100644 llmpool/health_order_test.go diff --git a/cmd/artex/main.go b/cmd/artex/main.go index 6595a281b..369721fc3 100644 --- a/cmd/artex/main.go +++ b/cmd/artex/main.go @@ -142,6 +142,9 @@ func run() int { if drainErr != nil { log.Printf("task drain: %v", drainErr) } + // The health writer flushes on ctx.Done(). Wait for it before the deferred + // store close, so the last cooling window is not written at a closed database. + srv.FlushLLMHealth() if httpErr != nil || drainErr != nil { return 1 } diff --git a/llmpool/health.go b/llmpool/health.go index 017f6c88b..b79b26bdb 100644 --- a/llmpool/health.go +++ b/llmpool/health.go @@ -46,7 +46,9 @@ type Registry struct { m map[int64]*State // persist / forget mirror state to the DB so a cooling-off window survives a - // restart. Both may be nil (no DB); both are called off the hot path. + // restart. Both may be nil (no DB). Both run while r.mu is held, so a later + // change to the same profile cannot enqueue ahead of this one. They must not + // call back into the registry. persist func(id int64, st State) forget func(id int64) @@ -118,16 +120,14 @@ func (r *Registry) IsOpen(id int64) bool { return r.Get(id).Open() } // recovers is fully trusted again (and drops the persisted row). func (r *Registry) Pass(id int64) { r.mu.Lock() + defer r.mu.Unlock() st := r.m[id] if st == nil || (st.Fails == 0 && st.Trips == 0 && st.OpenUntil.IsZero()) { - r.mu.Unlock() - return // already clean — nothing to write + return // already clean; nothing to write } delete(r.m, id) - forget := r.forget - r.mu.Unlock() - if forget != nil { - forget(id) + if r.forget != nil { + r.forget(id) } } @@ -137,6 +137,7 @@ func (r *Registry) Pass(id int64) { // true when this call is what opened the breaker, so the caller can log it once. func (r *Registry) Trip(id int64, errMsg string, hard bool) (tripped bool) { r.mu.Lock() + defer r.mu.Unlock() st := r.m[id] if st == nil { st = &State{} @@ -152,11 +153,8 @@ func (r *Registry) Trip(id int64, errMsg string, hard bool) (tripped bool) { st.Fails = 0 // counted into this trip; start fresh for the half-open probe tripped = true } - snap := *st - persist := r.persist - r.mu.Unlock() - if persist != nil { - persist(id, snap) + if r.persist != nil { + r.persist(id, *st) } return tripped } @@ -164,11 +162,10 @@ func (r *Registry) Trip(id int64, errMsg string, hard bool) (tripped bool) { // Reset clears one profile's state — the UI's "Recover now" action. func (r *Registry) Reset(id int64) { r.mu.Lock() + defer r.mu.Unlock() delete(r.m, id) - forget := r.forget - r.mu.Unlock() - if forget != nil { - forget(id) + if r.forget != nil { + r.forget(id) } } diff --git a/llmpool/health_order_test.go b/llmpool/health_order_test.go new file mode 100644 index 000000000..576722494 --- /dev/null +++ b/llmpool/health_order_test.go @@ -0,0 +1,51 @@ +package llmpool + +import ( + "sync/atomic" + "testing" + "time" +) + +// persist and forget run under the registry lock. A second update on the same +// profile cannot enqueue until the first callback returns, so the queue cannot +// observe the later state first. +func TestHealthCallbacksStayInStateOrder(t *testing.T) { + var entered atomic.Int32 + firstIn := make(chan struct{}) + release := make(chan struct{}) + block := func() { + if entered.Add(1) == 1 { + close(firstIn) + <-release + } + } + reg := NewRegistry( + func(int64, State) { block() }, + func(int64) { block() }, + ) + go reg.Trip(1, "down", true) + select { + case <-firstIn: + case <-time.After(2 * time.Second): + t.Fatal("Trip did not reach persist") + } + passed := make(chan struct{}) + go func() { + reg.Pass(1) + close(passed) + }() + select { + case <-passed: + t.Fatal("Pass enqueued while Trip's persist was still running") + case <-time.After(50 * time.Millisecond): + } + close(release) + select { + case <-passed: + case <-time.After(2 * time.Second): + t.Fatal("Pass did not finish after Trip released the lock") + } + if entered.Load() != 2 { + t.Fatalf("callbacks=%d, want Trip then Pass", entered.Load()) + } +} diff --git a/server/llmpool.go b/server/llmpool.go index 65b946522..087723a17 100644 --- a/server/llmpool.go +++ b/server/llmpool.go @@ -1,6 +1,7 @@ package server import ( + "context" "log" "sync" "time" @@ -18,13 +19,14 @@ import ( // - Circuit-breaker state is process-wide (s.llmHealth); rebuilding the pool does not clear it. // newLLMHealthRegistry builds the process-wide circuit-breaker registry, mirroring -// state into PG so a cooling-off window survives a restart. Writes are async and -// best-effort — the in-memory copy is authoritative. -func newLLMHealthRegistry(pg *db.DB) *llmpool.Registry { +// state into PG so a cooling-off window survives a restart. Writes are async. +// The in-memory copy is authoritative. The returned syncer is nil when there is +// no database; otherwise it flushes when ctx is cancelled. +func newLLMHealthRegistry(ctx context.Context, pg *db.DB) (*llmpool.Registry, *healthSync) { if pg == nil { - return llmpool.NewRegistry(nil, nil) + return llmpool.NewRegistry(nil, nil), nil } - syncer := newHealthSync(pg.SaveLLMHealth, pg.ClearLLMHealth) + syncer := newHealthSync(ctx, pg.SaveLLMHealth, pg.ClearLLMHealth) persist := func(id int64, st llmpool.State) { syncer.enqueue(id, healthOp{st: st}) } @@ -44,11 +46,13 @@ func newLLMHealthRegistry(pg *db.DB) *llmpool.Registry { log.Printf("[llmpool] restored circuit-breaker state: profile #%d cooling until %s", h.ProfileID, st.OpenUntil.Format(time.RFC3339)) } } - return reg + return reg, syncer } -// healthOp is the latest circuit-breaker write for one profile. clear drops the row. +// healthOp is one circuit-breaker write for a profile. clear drops the row. +// seq orders writes for that profile: a newer op wins over one still in flight. type healthOp struct { + seq uint64 clear bool st llmpool.State } @@ -56,35 +60,67 @@ type healthOp struct { // healthSync writes circuit-breaker state from one goroutine. A newer update for // the same profile replaces one that has not been written yet, so a burst of // state changes becomes one database call and the request path does not wait. +// save and clear must not call enqueue. type healthSync struct { - save func(db.LLMHealth) error - clear func(int64) error - mu sync.Mutex - pending map[int64]healthOp - wake chan struct{} + save func(db.LLMHealth) error + clear func(int64) error + retryAfter time.Duration // tests set a short delay; zero uses one second + mu sync.Mutex + pending map[int64]healthOp + applied map[int64]uint64 + seq uint64 + stopped bool + wake chan struct{} + done chan struct{} } -func newHealthSync(save func(db.LLMHealth) error, clear func(int64) error) *healthSync { +func newHealthSync(ctx context.Context, save func(db.LLMHealth) error, clear func(int64) error) *healthSync { + if ctx == nil { + ctx = context.Background() + } h := &healthSync{ save: save, clear: clear, pending: map[int64]healthOp{}, + applied: map[int64]uint64{}, wake: make(chan struct{}, 1), + done: make(chan struct{}), } - go h.loop() + go h.loop(ctx) return h } +// FlushLLMHealth waits until the circuit-breaker writer has flushed the ops +// queued before shutdown. Call it after the server context is cancelled and +// before the database closes. +func (s *Server) FlushLLMHealth() { + if s == nil || s.llmHealthSync == nil { + return + } + select { + case <-s.llmHealthSync.done: + case <-time.After(5 * time.Second): + log.Printf("[llmpool] timed out flushing circuit-breaker state") + } +} + func (h *healthSync) enqueue(id int64, op healthOp) { h.mu.Lock() - if h.pending == nil { - h.pending = map[int64]healthOp{} + h.seq++ + op.seq = h.seq + if h.stopped { + h.mu.Unlock() + if err := h.apply(id, op); err != nil { + log.Printf("[llmpool] write circuit-breaker profile #%d: %v", id, err) + } + return } - h.pending[id] = op - h.mu.Unlock() - if h.wake == nil { + if prev, ok := h.pending[id]; ok && prev.seq > op.seq { + h.mu.Unlock() return } + h.pending[id] = op + h.mu.Unlock() select { case h.wake <- struct{}{}: default: @@ -102,34 +138,88 @@ func (h *healthSync) take() map[int64]healthOp { return out } -func (h *healthSync) loop() { +func (h *healthSync) restore(id int64, op healthOp) { + h.mu.Lock() + defer h.mu.Unlock() + if h.stopped || h.applied[id] >= op.seq { + return + } + if cur, ok := h.pending[id]; ok && cur.seq >= op.seq { + return + } + h.pending[id] = op +} + +func (h *healthSync) apply(id int64, op healthOp) error { + h.mu.Lock() + defer h.mu.Unlock() + if op.seq > 0 && h.applied[id] >= op.seq { + return nil + } + var err error + if op.clear { + if h.clear != nil { + err = h.clear(id) + } + } else if h.save != nil { + row := db.LLMHealth{ProfileID: id, Fails: op.st.Fails, Trips: op.st.Trips, LastError: op.st.LastError} + if !op.st.OpenUntil.IsZero() { + openUntil := op.st.OpenUntil + row.OpenUntil = &openUntil + } + err = h.save(row) + } + if err != nil { + return err + } + if h.applied[id] < op.seq { + h.applied[id] = op.seq + } + return nil +} + +func (h *healthSync) write(batch map[int64]healthOp) (failed bool) { + for id, op := range batch { + if err := h.apply(id, op); err != nil { + log.Printf("[llmpool] write circuit-breaker profile #%d: %v", id, err) + h.restore(id, op) + failed = true + } + } + return failed +} + +func (h *healthSync) finish() { + h.mu.Lock() + h.stopped = true + batch := h.pending + h.pending = map[int64]healthOp{} + h.mu.Unlock() + for id, op := range batch { + if err := h.apply(id, op); err != nil { + log.Printf("[llmpool] write circuit-breaker profile #%d: %v", id, err) + } + } +} + +func (h *healthSync) loop(ctx context.Context) { + defer close(h.done) for { - batch := h.take() - if len(batch) == 0 { - <-h.wake - continue + failed := h.write(h.take()) + delay := h.retryAfter + if delay <= 0 { + delay = time.Second } - for id, op := range batch { - if op.clear { - if h.clear == nil { - continue - } - if err := h.clear(id); err != nil { - log.Printf("[llmpool] clear circuit-breaker profile #%d: %v", id, err) - } - continue - } - if h.save == nil { - continue - } - row := db.LLMHealth{ProfileID: id, Fails: op.st.Fails, Trips: op.st.Trips, LastError: op.st.LastError} - if !op.st.OpenUntil.IsZero() { - openUntil := op.st.OpenUntil - row.OpenUntil = &openUntil - } - if err := h.save(row); err != nil { - log.Printf("[llmpool] persist circuit-breaker profile #%d: %v", id, err) - } + var retry <-chan time.Time + if failed { + retry = time.After(delay) + } + select { + case <-ctx.Done(): + h.finish() + return + case <-h.wake: + case <-retry: } } } diff --git a/server/llmpool_health_test.go b/server/llmpool_health_test.go index c759d2c08..9d14b6396 100644 --- a/server/llmpool_health_test.go +++ b/server/llmpool_health_test.go @@ -1,51 +1,110 @@ package server import ( + "context" + "errors" + "sync" "testing" + "time" "github.com/Autumn-27/artex/db" "github.com/Autumn-27/artex/llmpool" ) -func TestHealthSyncCoalescesToLatest(t *testing.T) { - h := &healthSync{pending: map[int64]healthOp{}} - h.enqueue(7, healthOp{st: llmpool.State{Fails: 1}}) - h.enqueue(7, healthOp{st: llmpool.State{Fails: 4}}) - h.enqueue(7, healthOp{clear: true}) - h.enqueue(8, healthOp{st: llmpool.State{Fails: 2}}) - batch := h.take() - if len(batch) != 2 || !batch[7].clear || batch[8].st.Fails != 2 { - t.Fatalf("batch=%+v", batch) - } - if again := h.take(); again != nil { - t.Fatalf("pending was not cleared: %+v", again) - } - var saved []db.LLMHealth - var cleared []int64 - worker := &healthSync{ - save: func(row db.LLMHealth) error { - saved = append(saved, row) +func TestHealthSyncLoopWritesLatest(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var mu sync.Mutex + saved := map[int64]db.LLMHealth{} + cleared := map[int64]int{} + savedCh := make(chan struct{}, 4) + clearedCh := make(chan struct{}, 4) + h := newHealthSync(ctx, + func(row db.LLMHealth) error { + mu.Lock() + saved[row.ProfileID] = row + mu.Unlock() + savedCh <- struct{}{} return nil }, - clear: func(id int64) error { - cleared = append(cleared, id) + func(id int64) error { + mu.Lock() + cleared[id]++ + delete(saved, id) + mu.Unlock() + clearedCh <- struct{}{} return nil }, - } - worker.enqueue(7, healthOp{clear: true}) - worker.enqueue(8, healthOp{st: llmpool.State{Fails: 3, LastError: "down"}}) - for id, op := range worker.take() { - if op.clear { - if err := worker.clear(id); err != nil { - t.Fatal(err) - } - continue + ) + openUntil := time.Now().Add(time.Minute).UTC().Truncate(time.Second) + h.enqueue(8, healthOp{st: llmpool.State{Fails: 1, LastError: "first"}}) + h.enqueue(8, healthOp{st: llmpool.State{Fails: 4, Trips: 1, OpenUntil: openUntil, LastError: "down"}}) + deadline := time.After(2 * time.Second) + var latest db.LLMHealth + for { + mu.Lock() + latest = saved[8] + mu.Unlock() + if latest.Fails == 4 && latest.OpenUntil != nil && latest.OpenUntil.Equal(openUntil) { + break } - if err := worker.save(db.LLMHealth{ProfileID: id, Fails: op.st.Fails, LastError: op.st.LastError}); err != nil { - t.Fatal(err) + select { + case <-savedCh: + case <-deadline: + t.Fatalf("loop saved=%+v, want the latest trip", latest) } } - if len(cleared) != 1 || cleared[0] != 7 || len(saved) != 1 || saved[0].ProfileID != 8 || saved[0].Fails != 3 { - t.Fatalf("cleared=%v saved=%+v", cleared, saved) + h.enqueue(7, healthOp{clear: true}) + select { + case <-clearedCh: + case <-time.After(2 * time.Second): + t.Fatal("loop did not clear") + } + mu.Lock() + defer mu.Unlock() + got := saved[8] + if got.Fails != 4 || got.Trips != 1 || got.LastError != "down" || got.OpenUntil == nil || !got.OpenUntil.Equal(openUntil) { + t.Fatalf("saved=%+v, want the latest trip including OpenUntil", got) + } + if cleared[7] != 1 { + t.Fatalf("cleared=%v", cleared) + } +} + +func TestHealthSyncRetriesFailedSave(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var mu sync.Mutex + calls := 0 + done := make(chan struct{}) + h := newHealthSync(ctx, + func(row db.LLMHealth) error { + mu.Lock() + calls++ + n := calls + mu.Unlock() + if n == 1 { + return errors.New("database down") + } + if n == 2 { + close(done) + } + return nil + }, + func(int64) error { return nil }, + ) + h.retryAfter = 20 * time.Millisecond + h.enqueue(3, healthOp{st: llmpool.State{Fails: 2, LastError: "blip"}}) + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("failed save was not retried") + } + mu.Lock() + defer mu.Unlock() + if calls < 2 { + t.Fatalf("save calls=%d, want a retry", calls) } } diff --git a/server/server.go b/server/server.go index aa8872882..58f374c3d 100644 --- a/server/server.go +++ b/server/server.go @@ -117,7 +117,9 @@ type Server struct { // It deliberately lives OUTSIDE the provider caches: rebuilding the chain // (saving an unrelated profile, flipping a setting) must not erase what we // learned about which backends are out of credit / rate-limited. - llmHealth *llmpool.Registry + // llmHealthSync flushes that state when ctx is cancelled, before the database closes. + llmHealth *llmpool.Registry + llmHealthSync *healthSync // Explicit task chains use one task-scoped dynamic provider shared by goals, // planner, workers, and the main agent. Bundles are stable; their runtime reads @@ -163,9 +165,10 @@ func New(ctx context.Context, m *Manager, skillDir string, dataDir string, keyDi chatCancel: map[string]context.CancelCauseFunc{}, triggerQ: map[string][]triggeredRun{}, triggerActive: map[string]int{}, triggerCfg: map[string]triggerBehavior{}, profChatAgents: map[int64]*agent.ChatAgent{}, - provByProfile: map[int64]*provEntry{}, llmHealth: newLLMHealthRegistry(m.pg), + provByProfile: map[int64]*provEntry{}, taskAgents: map[string]*taskAgentBundle{}, archiveWake: make(chan struct{}, 1), loginLimit: newLoginLimiter(time.Now)} + s.llmHealth, s.llmHealthSync = newLLMHealthRegistry(ctx, m.pg) for _, o := range opts { o(s) } From b2a2f95e8c25b55038ba63ef9299da377f24b9a0 Mon Sep 17 00:00:00 2001 From: protonmatter Date: Tue, 17 Nov 2026 16:27:14 -0500 Subject: [PATCH 3/4] Release the circuit-breaker queue lock around the database write. --- llmpool/health.go | 2 +- server/llmpool.go | 25 ++++++++++-- server/llmpool_health_test.go | 73 +++++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 5 deletions(-) diff --git a/llmpool/health.go b/llmpool/health.go index b79b26bdb..7f3e3015e 100644 --- a/llmpool/health.go +++ b/llmpool/health.go @@ -48,7 +48,7 @@ type Registry struct { // persist / forget mirror state to the DB so a cooling-off window survives a // restart. Both may be nil (no DB). Both run while r.mu is held, so a later // change to the same profile cannot enqueue ahead of this one. They must not - // call back into the registry. + // call back into the registry or wait on database I/O. persist func(id int64, st State) forget func(id int64) diff --git a/server/llmpool.go b/server/llmpool.go index 087723a17..bbb1d1c8f 100644 --- a/server/llmpool.go +++ b/server/llmpool.go @@ -59,13 +59,16 @@ type healthOp struct { // healthSync writes circuit-breaker state from one goroutine. A newer update for // the same profile replaces one that has not been written yet, so a burst of -// state changes becomes one database call and the request path does not wait. -// save and clear must not call enqueue. +// state changes becomes one database call. The database call runs without mu, +// so enqueue does not wait on a slow write. writeMu keeps those calls in order. +// Before each call, an op that is already older than the applied or pending +// sequence is dropped. save and clear must not call enqueue. type healthSync struct { save func(db.LLMHealth) error clear func(int64) error retryAfter time.Duration // tests set a short delay; zero uses one second mu sync.Mutex + writeMu sync.Mutex // orders database calls; enqueue does not take it pending map[int64]healthOp applied map[int64]uint64 seq uint64 @@ -150,10 +153,22 @@ func (h *healthSync) restore(id int64, op healthOp) { h.pending[id] = op } -func (h *healthSync) apply(id int64, op healthOp) error { +// superseded reports whether a newer op for id is already applied or waiting. +// The caller holds writeMu, so the decision and the database call stay in order. +func (h *healthSync) superseded(id int64, seq uint64) bool { h.mu.Lock() defer h.mu.Unlock() - if op.seq > 0 && h.applied[id] >= op.seq { + if seq > 0 && h.applied[id] >= seq { + return true + } + cur, ok := h.pending[id] + return ok && cur.seq > seq +} + +func (h *healthSync) apply(id int64, op healthOp) error { + h.writeMu.Lock() + defer h.writeMu.Unlock() + if h.superseded(id, op.seq) { return nil } var err error @@ -172,9 +187,11 @@ func (h *healthSync) apply(id int64, op healthOp) error { if err != nil { return err } + h.mu.Lock() if h.applied[id] < op.seq { h.applied[id] = op.seq } + h.mu.Unlock() return nil } diff --git a/server/llmpool_health_test.go b/server/llmpool_health_test.go index 9d14b6396..c9e7a1eb1 100644 --- a/server/llmpool_health_test.go +++ b/server/llmpool_health_test.go @@ -108,3 +108,76 @@ func TestHealthSyncRetriesFailedSave(t *testing.T) { t.Fatalf("save calls=%d, want a retry", calls) } } + +func TestHealthSyncEnqueueDoesNotWaitOnSave(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + started := make(chan struct{}) + release := make(chan struct{}) + later := make(chan struct{}) + h := newHealthSync(ctx, + func(row db.LLMHealth) error { + if row.Fails == 1 { + close(started) + <-release + return nil + } + if row.Fails == 9 { + close(later) + } + return nil + }, + func(int64) error { return nil }, + ) + h.enqueue(4, healthOp{st: llmpool.State{Fails: 1}}) + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("save did not start") + } + done := make(chan struct{}) + go func() { + h.enqueue(4, healthOp{st: llmpool.State{Fails: 9, LastError: "later"}}) + close(done) + }() + select { + case <-done: + case <-time.After(200 * time.Millisecond): + t.Fatal("enqueue blocked while a save was in flight") + } + close(release) + select { + case <-later: + case <-time.After(2 * time.Second): + t.Fatal("the later state was not written") + } +} + +func TestHealthSyncDropsStaleWrite(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + writes := 0 + h := newHealthSync(ctx, + func(db.LLMHealth) error { + writes++ + return nil + }, + func(int64) error { return nil }, + ) + h.mu.Lock() + h.seq = 4 + h.applied[5] = 2 + h.pending[6] = healthOp{seq: 4, clear: true} + h.mu.Unlock() + if err := h.apply(5, healthOp{seq: 1, st: llmpool.State{Fails: 1}}); err != nil { + t.Fatal(err) + } + if err := h.apply(6, healthOp{seq: 3, st: llmpool.State{Fails: 3}}); err != nil { + t.Fatal(err) + } + if writes != 0 { + t.Fatalf("stale writes=%d", writes) + } +} From c9d81a453f176e673a95f11e6d0b59b6e31c84d3 Mon Sep 17 00:00:00 2001 From: protonmatter Date: Tue, 17 Nov 2026 16:38:35 -0500 Subject: [PATCH 4/4] Align the Server literal so gofmt accepts it. --- server/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index 58f374c3d..20b4d90b8 100644 --- a/server/server.go +++ b/server/server.go @@ -166,7 +166,7 @@ func New(ctx context.Context, m *Manager, skillDir string, dataDir string, keyDi triggerActive: map[string]int{}, triggerCfg: map[string]triggerBehavior{}, profChatAgents: map[int64]*agent.ChatAgent{}, provByProfile: map[int64]*provEntry{}, - taskAgents: map[string]*taskAgentBundle{}, archiveWake: make(chan struct{}, 1), + taskAgents: map[string]*taskAgentBundle{}, archiveWake: make(chan struct{}, 1), loginLimit: newLoginLimiter(time.Now)} s.llmHealth, s.llmHealthSync = newLLMHealthRegistry(ctx, m.pg) for _, o := range opts {