diff --git a/cmd/artex/main.go b/cmd/artex/main.go index 6595a281b..369721fc3 100644 --- a/cmd/artex/main.go +++ b/cmd/artex/main.go @@ -142,6 +142,9 @@ func run() int { if drainErr != nil { log.Printf("task drain: %v", drainErr) } + // The health writer flushes on ctx.Done(). Wait for it before the deferred + // store close, so the last cooling window is not written at a closed database. + srv.FlushLLMHealth() if httpErr != nil || drainErr != nil { return 1 } diff --git a/llmpool/health.go b/llmpool/health.go index 017f6c88b..7f3e3015e 100644 --- a/llmpool/health.go +++ b/llmpool/health.go @@ -46,7 +46,9 @@ type Registry struct { m map[int64]*State // persist / forget mirror state to the DB so a cooling-off window survives a - // restart. Both may be nil (no DB); both are called off the hot path. + // restart. Both may be nil (no DB). Both run while r.mu is held, so a later + // change to the same profile cannot enqueue ahead of this one. They must not + // call back into the registry or wait on database I/O. persist func(id int64, st State) forget func(id int64) @@ -118,16 +120,14 @@ func (r *Registry) IsOpen(id int64) bool { return r.Get(id).Open() } // recovers is fully trusted again (and drops the persisted row). func (r *Registry) Pass(id int64) { r.mu.Lock() + defer r.mu.Unlock() st := r.m[id] if st == nil || (st.Fails == 0 && st.Trips == 0 && st.OpenUntil.IsZero()) { - r.mu.Unlock() - return // already clean — nothing to write + return // already clean; nothing to write } delete(r.m, id) - forget := r.forget - r.mu.Unlock() - if forget != nil { - forget(id) + if r.forget != nil { + r.forget(id) } } @@ -137,6 +137,7 @@ func (r *Registry) Pass(id int64) { // true when this call is what opened the breaker, so the caller can log it once. func (r *Registry) Trip(id int64, errMsg string, hard bool) (tripped bool) { r.mu.Lock() + defer r.mu.Unlock() st := r.m[id] if st == nil { st = &State{} @@ -152,11 +153,8 @@ func (r *Registry) Trip(id int64, errMsg string, hard bool) (tripped bool) { st.Fails = 0 // counted into this trip; start fresh for the half-open probe tripped = true } - snap := *st - persist := r.persist - r.mu.Unlock() - if persist != nil { - persist(id, snap) + if r.persist != nil { + r.persist(id, *st) } return tripped } @@ -164,11 +162,10 @@ func (r *Registry) Trip(id int64, errMsg string, hard bool) (tripped bool) { // Reset clears one profile's state — the UI's "Recover now" action. func (r *Registry) Reset(id int64) { r.mu.Lock() + defer r.mu.Unlock() delete(r.m, id) - forget := r.forget - r.mu.Unlock() - if forget != nil { - forget(id) + if r.forget != nil { + r.forget(id) } } diff --git a/llmpool/health_order_test.go b/llmpool/health_order_test.go new file mode 100644 index 000000000..576722494 --- /dev/null +++ b/llmpool/health_order_test.go @@ -0,0 +1,51 @@ +package llmpool + +import ( + "sync/atomic" + "testing" + "time" +) + +// persist and forget run under the registry lock. A second update on the same +// profile cannot enqueue until the first callback returns, so the queue cannot +// observe the later state first. +func TestHealthCallbacksStayInStateOrder(t *testing.T) { + var entered atomic.Int32 + firstIn := make(chan struct{}) + release := make(chan struct{}) + block := func() { + if entered.Add(1) == 1 { + close(firstIn) + <-release + } + } + reg := NewRegistry( + func(int64, State) { block() }, + func(int64) { block() }, + ) + go reg.Trip(1, "down", true) + select { + case <-firstIn: + case <-time.After(2 * time.Second): + t.Fatal("Trip did not reach persist") + } + passed := make(chan struct{}) + go func() { + reg.Pass(1) + close(passed) + }() + select { + case <-passed: + t.Fatal("Pass enqueued while Trip's persist was still running") + case <-time.After(50 * time.Millisecond): + } + close(release) + select { + case <-passed: + case <-time.After(2 * time.Second): + t.Fatal("Pass did not finish after Trip released the lock") + } + if entered.Load() != 2 { + t.Fatalf("callbacks=%d, want Trip then Pass", entered.Load()) + } +} diff --git a/server/llmpool.go b/server/llmpool.go index a3901fcfa..bbb1d1c8f 100644 --- a/server/llmpool.go +++ b/server/llmpool.go @@ -1,7 +1,9 @@ package server import ( + "context" "log" + "sync" "time" "github.com/Autumn-27/artex/agent" @@ -17,26 +19,19 @@ import ( // - Circuit-breaker state is process-wide (s.llmHealth); rebuilding the pool does not clear it. // newLLMHealthRegistry builds the process-wide circuit-breaker registry, mirroring -// state into PG so a cooling-off window survives a restart. Writes are async and -// best-effort — the in-memory copy is authoritative. -func newLLMHealthRegistry(pg *db.DB) *llmpool.Registry { +// state into PG so a cooling-off window survives a restart. Writes are async. +// The in-memory copy is authoritative. The returned syncer is nil when there is +// no database; otherwise it flushes when ctx is cancelled. +func newLLMHealthRegistry(ctx context.Context, pg *db.DB) (*llmpool.Registry, *healthSync) { if pg == nil { - return llmpool.NewRegistry(nil, nil) + return llmpool.NewRegistry(nil, nil), nil } + syncer := newHealthSync(ctx, pg.SaveLLMHealth, pg.ClearLLMHealth) persist := func(id int64, st llmpool.State) { - h := db.LLMHealth{ProfileID: id, Fails: st.Fails, Trips: st.Trips, LastError: st.LastError} - if !st.OpenUntil.IsZero() { - t := st.OpenUntil - h.OpenUntil = &t - } - go func() { - if err := pg.SaveLLMHealth(h); err != nil { - log.Printf("[llmpool] failed to persist circuit-breaker state: %v", err) - } - }() + syncer.enqueue(id, healthOp{st: st}) } forget := func(id int64) { - go func() { _ = pg.ClearLLMHealth(id) }() + syncer.enqueue(id, healthOp{clear: true}) } reg := llmpool.NewRegistry(persist, forget) // Restore only windows that haven't expired (LoadLLMHealth filters), so a @@ -51,7 +46,199 @@ func newLLMHealthRegistry(pg *db.DB) *llmpool.Registry { log.Printf("[llmpool] restored circuit-breaker state: profile #%d cooling until %s", h.ProfileID, st.OpenUntil.Format(time.RFC3339)) } } - return reg + return reg, syncer +} + +// healthOp is one circuit-breaker write for a profile. clear drops the row. +// seq orders writes for that profile: a newer op wins over one still in flight. +type healthOp struct { + seq uint64 + clear bool + st llmpool.State +} + +// healthSync writes circuit-breaker state from one goroutine. A newer update for +// the same profile replaces one that has not been written yet, so a burst of +// state changes becomes one database call. The database call runs without mu, +// so enqueue does not wait on a slow write. writeMu keeps those calls in order. +// Before each call, an op that is already older than the applied or pending +// sequence is dropped. save and clear must not call enqueue. +type healthSync struct { + save func(db.LLMHealth) error + clear func(int64) error + retryAfter time.Duration // tests set a short delay; zero uses one second + mu sync.Mutex + writeMu sync.Mutex // orders database calls; enqueue does not take it + pending map[int64]healthOp + applied map[int64]uint64 + seq uint64 + stopped bool + wake chan struct{} + done chan struct{} +} + +func newHealthSync(ctx context.Context, save func(db.LLMHealth) error, clear func(int64) error) *healthSync { + if ctx == nil { + ctx = context.Background() + } + h := &healthSync{ + save: save, + clear: clear, + pending: map[int64]healthOp{}, + applied: map[int64]uint64{}, + wake: make(chan struct{}, 1), + done: make(chan struct{}), + } + go h.loop(ctx) + return h +} + +// FlushLLMHealth waits until the circuit-breaker writer has flushed the ops +// queued before shutdown. Call it after the server context is cancelled and +// before the database closes. +func (s *Server) FlushLLMHealth() { + if s == nil || s.llmHealthSync == nil { + return + } + select { + case <-s.llmHealthSync.done: + case <-time.After(5 * time.Second): + log.Printf("[llmpool] timed out flushing circuit-breaker state") + } +} + +func (h *healthSync) enqueue(id int64, op healthOp) { + h.mu.Lock() + h.seq++ + op.seq = h.seq + if h.stopped { + h.mu.Unlock() + if err := h.apply(id, op); err != nil { + log.Printf("[llmpool] write circuit-breaker profile #%d: %v", id, err) + } + return + } + if prev, ok := h.pending[id]; ok && prev.seq > op.seq { + h.mu.Unlock() + return + } + h.pending[id] = op + h.mu.Unlock() + select { + case h.wake <- struct{}{}: + default: + } +} + +func (h *healthSync) take() map[int64]healthOp { + h.mu.Lock() + defer h.mu.Unlock() + if len(h.pending) == 0 { + return nil + } + out := h.pending + h.pending = map[int64]healthOp{} + return out +} + +func (h *healthSync) restore(id int64, op healthOp) { + h.mu.Lock() + defer h.mu.Unlock() + if h.stopped || h.applied[id] >= op.seq { + return + } + if cur, ok := h.pending[id]; ok && cur.seq >= op.seq { + return + } + h.pending[id] = op +} + +// superseded reports whether a newer op for id is already applied or waiting. +// The caller holds writeMu, so the decision and the database call stay in order. +func (h *healthSync) superseded(id int64, seq uint64) bool { + h.mu.Lock() + defer h.mu.Unlock() + if seq > 0 && h.applied[id] >= seq { + return true + } + cur, ok := h.pending[id] + return ok && cur.seq > seq +} + +func (h *healthSync) apply(id int64, op healthOp) error { + h.writeMu.Lock() + defer h.writeMu.Unlock() + if h.superseded(id, op.seq) { + return nil + } + var err error + if op.clear { + if h.clear != nil { + err = h.clear(id) + } + } else if h.save != nil { + row := db.LLMHealth{ProfileID: id, Fails: op.st.Fails, Trips: op.st.Trips, LastError: op.st.LastError} + if !op.st.OpenUntil.IsZero() { + openUntil := op.st.OpenUntil + row.OpenUntil = &openUntil + } + err = h.save(row) + } + if err != nil { + return err + } + h.mu.Lock() + if h.applied[id] < op.seq { + h.applied[id] = op.seq + } + h.mu.Unlock() + return nil +} + +func (h *healthSync) write(batch map[int64]healthOp) (failed bool) { + for id, op := range batch { + if err := h.apply(id, op); err != nil { + log.Printf("[llmpool] write circuit-breaker profile #%d: %v", id, err) + h.restore(id, op) + failed = true + } + } + return failed +} + +func (h *healthSync) finish() { + h.mu.Lock() + h.stopped = true + batch := h.pending + h.pending = map[int64]healthOp{} + h.mu.Unlock() + for id, op := range batch { + if err := h.apply(id, op); err != nil { + log.Printf("[llmpool] write circuit-breaker profile #%d: %v", id, err) + } + } +} + +func (h *healthSync) loop(ctx context.Context) { + defer close(h.done) + for { + failed := h.write(h.take()) + delay := h.retryAfter + if delay <= 0 { + delay = time.Second + } + var retry <-chan time.Time + if failed { + retry = time.After(delay) + } + select { + case <-ctx.Done(): + h.finish() + return + case <-h.wake: + case <-retry: + } + } } // poolMember builds one chain member from a profile, reusing the per-profile diff --git a/server/llmpool_health_test.go b/server/llmpool_health_test.go new file mode 100644 index 000000000..c9e7a1eb1 --- /dev/null +++ b/server/llmpool_health_test.go @@ -0,0 +1,183 @@ +package server + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/artex/llmpool" +) + +func TestHealthSyncLoopWritesLatest(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var mu sync.Mutex + saved := map[int64]db.LLMHealth{} + cleared := map[int64]int{} + savedCh := make(chan struct{}, 4) + clearedCh := make(chan struct{}, 4) + h := newHealthSync(ctx, + func(row db.LLMHealth) error { + mu.Lock() + saved[row.ProfileID] = row + mu.Unlock() + savedCh <- struct{}{} + return nil + }, + func(id int64) error { + mu.Lock() + cleared[id]++ + delete(saved, id) + mu.Unlock() + clearedCh <- struct{}{} + return nil + }, + ) + openUntil := time.Now().Add(time.Minute).UTC().Truncate(time.Second) + h.enqueue(8, healthOp{st: llmpool.State{Fails: 1, LastError: "first"}}) + h.enqueue(8, healthOp{st: llmpool.State{Fails: 4, Trips: 1, OpenUntil: openUntil, LastError: "down"}}) + deadline := time.After(2 * time.Second) + var latest db.LLMHealth + for { + mu.Lock() + latest = saved[8] + mu.Unlock() + if latest.Fails == 4 && latest.OpenUntil != nil && latest.OpenUntil.Equal(openUntil) { + break + } + select { + case <-savedCh: + case <-deadline: + t.Fatalf("loop saved=%+v, want the latest trip", latest) + } + } + h.enqueue(7, healthOp{clear: true}) + select { + case <-clearedCh: + case <-time.After(2 * time.Second): + t.Fatal("loop did not clear") + } + mu.Lock() + defer mu.Unlock() + got := saved[8] + if got.Fails != 4 || got.Trips != 1 || got.LastError != "down" || got.OpenUntil == nil || !got.OpenUntil.Equal(openUntil) { + t.Fatalf("saved=%+v, want the latest trip including OpenUntil", got) + } + if cleared[7] != 1 { + t.Fatalf("cleared=%v", cleared) + } +} + +func TestHealthSyncRetriesFailedSave(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var mu sync.Mutex + calls := 0 + done := make(chan struct{}) + h := newHealthSync(ctx, + func(row db.LLMHealth) error { + mu.Lock() + calls++ + n := calls + mu.Unlock() + if n == 1 { + return errors.New("database down") + } + if n == 2 { + close(done) + } + return nil + }, + func(int64) error { return nil }, + ) + h.retryAfter = 20 * time.Millisecond + h.enqueue(3, healthOp{st: llmpool.State{Fails: 2, LastError: "blip"}}) + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("failed save was not retried") + } + mu.Lock() + defer mu.Unlock() + if calls < 2 { + t.Fatalf("save calls=%d, want a retry", calls) + } +} + +func TestHealthSyncEnqueueDoesNotWaitOnSave(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + started := make(chan struct{}) + release := make(chan struct{}) + later := make(chan struct{}) + h := newHealthSync(ctx, + func(row db.LLMHealth) error { + if row.Fails == 1 { + close(started) + <-release + return nil + } + if row.Fails == 9 { + close(later) + } + return nil + }, + func(int64) error { return nil }, + ) + h.enqueue(4, healthOp{st: llmpool.State{Fails: 1}}) + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("save did not start") + } + done := make(chan struct{}) + go func() { + h.enqueue(4, healthOp{st: llmpool.State{Fails: 9, LastError: "later"}}) + close(done) + }() + select { + case <-done: + case <-time.After(200 * time.Millisecond): + t.Fatal("enqueue blocked while a save was in flight") + } + close(release) + select { + case <-later: + case <-time.After(2 * time.Second): + t.Fatal("the later state was not written") + } +} + +func TestHealthSyncDropsStaleWrite(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + writes := 0 + h := newHealthSync(ctx, + func(db.LLMHealth) error { + writes++ + return nil + }, + func(int64) error { return nil }, + ) + h.mu.Lock() + h.seq = 4 + h.applied[5] = 2 + h.pending[6] = healthOp{seq: 4, clear: true} + h.mu.Unlock() + if err := h.apply(5, healthOp{seq: 1, st: llmpool.State{Fails: 1}}); err != nil { + t.Fatal(err) + } + if err := h.apply(6, healthOp{seq: 3, st: llmpool.State{Fails: 3}}); err != nil { + t.Fatal(err) + } + if writes != 0 { + t.Fatalf("stale writes=%d", writes) + } +} diff --git a/server/server.go b/server/server.go index aa8872882..20b4d90b8 100644 --- a/server/server.go +++ b/server/server.go @@ -117,7 +117,9 @@ type Server struct { // It deliberately lives OUTSIDE the provider caches: rebuilding the chain // (saving an unrelated profile, flipping a setting) must not erase what we // learned about which backends are out of credit / rate-limited. - llmHealth *llmpool.Registry + // llmHealthSync flushes that state when ctx is cancelled, before the database closes. + llmHealth *llmpool.Registry + llmHealthSync *healthSync // Explicit task chains use one task-scoped dynamic provider shared by goals, // planner, workers, and the main agent. Bundles are stable; their runtime reads @@ -163,9 +165,10 @@ func New(ctx context.Context, m *Manager, skillDir string, dataDir string, keyDi chatCancel: map[string]context.CancelCauseFunc{}, triggerQ: map[string][]triggeredRun{}, triggerActive: map[string]int{}, triggerCfg: map[string]triggerBehavior{}, profChatAgents: map[int64]*agent.ChatAgent{}, - provByProfile: map[int64]*provEntry{}, llmHealth: newLLMHealthRegistry(m.pg), - taskAgents: map[string]*taskAgentBundle{}, archiveWake: make(chan struct{}, 1), + provByProfile: map[int64]*provEntry{}, + taskAgents: map[string]*taskAgentBundle{}, archiveWake: make(chan struct{}, 1), loginLimit: newLoginLimiter(time.Now)} + s.llmHealth, s.llmHealthSync = newLLMHealthRegistry(ctx, m.pg) for _, o := range opts { o(s) }