From 0317c1ab8adab34509d675f2beb1e2bf3f105d10 Mon Sep 17 00:00:00 2001 From: Protonmatter Date: Tue, 17 Nov 2026 13:42:44 +0000 Subject: [PATCH] Keep control-plane state correct when a write or a restart fails Queue an activity that could not be stored and tell the live stream the step failed. Rebuild spend caps from llm_usage, reject sessions issued before a password change, and wait for task goroutines on shutdown. A failed password read no longer mints a setup token, a coverage-flag error leaves tools available, and JSON bodies, login eviction, circuit-breaker writes, and MITM passthrough no longer fail open forever. --- agent/budget.go | 13 ++++ agent/budget_test.go | 11 +++ agent/coverage.go | 22 ++++++ agent/mainagent.go | 2 +- agent/planner.go | 2 +- agent/worker.go | 2 +- cmd/artex/main.go | 21 ++++- db/activity_outbox.go | 90 +++++++++++++++++++++ db/activity_outbox_test.go | 25 ++++++ db/exploration.go | 5 +- db/llm_usage.go | 11 +++ db/migrate_test.go | 4 +- db/migrations/0005_activity_outbox.sql | 14 ++++ db/task_scope.go | 19 +++-- server/auth.go | 104 +++++++++++++++++++++---- server/engine.go | 61 +++++++++++++++ server/engine_budget.go | 29 ++++++- server/engine_timeout.go | 4 + server/llmpool.go | 102 +++++++++++++++++++++--- server/login_limit.go | 26 ++++++- server/login_limit_test.go | 30 +++++++ server/reliability_test.go | 89 +++++++++++++++++++++ server/server.go | 41 +++++++++- server/server_mgmt.go | 10 ++- traffic/passthrough_test.go | 20 +++++ traffic/traffic.go | 33 ++++++-- web/src/lib/api.ts | 7 +- 27 files changed, 741 insertions(+), 56 deletions(-) create mode 100644 agent/coverage.go create mode 100644 db/activity_outbox.go create mode 100644 db/activity_outbox_test.go create mode 100644 db/migrations/0005_activity_outbox.sql create mode 100644 server/reliability_test.go diff --git a/agent/budget.go b/agent/budget.go index f22028e0..8fe727e5 100644 --- a/agent/budget.go +++ b/agent/budget.go @@ -110,6 +110,19 @@ func NewBudgetLedger(task, intent Budget) *BudgetLedger { return &BudgetLedger{task: task, intent: intent} } +// RestoreSpend adds usage that was already recorded before this process started. +// It is applied once, to a ledger that has not recorded anything yet. A nil +// ledger (no budget configured) is a no-op. +func (l *BudgetLedger) RestoreSpend(spent Spend) { + if l == nil || (spent.Tokens == 0 && spent.Turns == 0) { + return + } + l.mu.Lock() + l.spent.Tokens += spent.Tokens + l.spent.Turns += spent.Turns + l.mu.Unlock() +} + // Record meters one task-level call (the planner's own rounds). func (l *BudgetLedger) Record(u llm.Usage) { l.mu.Lock() diff --git a/agent/budget_test.go b/agent/budget_test.go index b35f3588..2c4c10a1 100644 --- a/agent/budget_test.go +++ b/agent/budget_test.go @@ -144,3 +144,14 @@ func TestBudgetCountsMatchUsageFixture(t *testing.T) { t.Fatalf("spent %+v remaining %+v estimate %+v, want {%d 5}, remaining, {10400 3}", st.Spent, st.Remaining, st.Estimate, want) } } + +func TestRestoreSpendCountsTowardCap(t *testing.T) { + l := NewBudgetLedger(NewBudget(100, 10), NewBudget(0, 0)) + l.RestoreSpend(Spend{Tokens: 40, Turns: 4}) + st := l.State() + if st.Spent != (Spend{40, 4}) || st.Remaining != (Budget{60, 6}) { + t.Fatalf("spent=%+v remaining=%+v", st.Spent, st.Remaining) + } + var none *BudgetLedger + none.RestoreSpend(Spend{Tokens: 1, Turns: 1}) +} diff --git a/agent/coverage.go b/agent/coverage.go new file mode 100644 index 00000000..6ad9b40f --- /dev/null +++ b/agent/coverage.go @@ -0,0 +1,22 @@ +package agent + +import ( + "log" + + "github.com/Autumn-27/artex/db" +) + +// coverageOn reports whether asset-coverage tools stay available for this task. +// A nil store is the historical default (enabled). A database error is logged and +// treated as enabled so a transient read failure does not drop the tools mid-run. +func coverageOn(as *db.AssetStore, taskID int64) bool { + if as == nil { + return true + } + enabled, err := as.CoverageEnabled(taskID) + if err != nil { + log.Printf("[coverage] task %d: reading the coverage flag failed; leaving coverage on: %v", taskID, err) + return true + } + return enabled +} diff --git a/agent/mainagent.go b/agent/mainagent.go index 3b8345b7..750089c8 100644 --- a/agent/mainagent.go +++ b/agent/mainagent.go @@ -118,7 +118,7 @@ func (m *MainAgent) Chat(ctx context.Context, taskID int64, mainSeg int, as *db. tsx.SetAssetStore(as, as.Companies()) } tsx.SetTaskID(taskID) - tsx.SetCoverageEnabled(as == nil || as.CoverageEnabled(taskID)) + tsx.SetCoverageEnabled(coverageOn(as, taskID)) tsx.SetNotify(notify) // generic wake-up (write ops without a dedicated callback go through it, debounced) tsx.SetResumeTask(resume) // set_goals adds a goal → pull a completed/paused task back to running tsx.SetNotifyGoal(notifyGoal) // set_goals adds a goal → record a "human added a goal: …" trigger for the planner diff --git a/agent/planner.go b/agent/planner.go index 55582447..dac72020 100644 --- a/agent/planner.go +++ b/agent/planner.go @@ -472,7 +472,7 @@ func (p *Planner) Plan(ctx context.Context, taskID int64, as *db.AssetStore, ts tsx.SetAssetStore(as, as.Companies()) } tsx.SetTaskID(taskID) - tsx.SetCoverageEnabled(as == nil || as.CoverageEnabled(taskID)) + tsx.SetCoverageEnabled(coverageOn(as, taskID)) tsx.killWork = p.killWork // enable kill_work tool (nil = unavailable) tsx.steerWork = p.steerWork // enable steer_work tool (nil = unavailable) if origin, _ := ts.OriginFactID(); origin > 0 { diff --git a/agent/worker.go b/agent/worker.go index b10cbeb5..1dc7dc9d 100644 --- a/agent/worker.go +++ b/agent/worker.go @@ -407,7 +407,7 @@ func (w *Worker) execute(ctx context.Context, name string, taskID int64, as *db. tsx := NewToolSet(ts, name) tsx.SetFindingRecorder(w.findingRecorder) tsx.SetTaskID(taskID) - coverageEnabled := as == nil || as.CoverageEnabled(taskID) + coverageEnabled := coverageOn(as, taskID) tsx.SetCoverageEnabled(coverageEnabled) if as != nil { tsx.SetAssetStore(as, as.Companies()) diff --git a/cmd/artex/main.go b/cmd/artex/main.go index 99e1180d..5aef97a2 100644 --- a/cmd/artex/main.go +++ b/cmd/artex/main.go @@ -110,10 +110,15 @@ func run() int { log.Printf("[config] trusted proxies: %v", trusted) } srv := server.New(ctx, mgr, skillDir, *dataDir, keyDir, server.WithSecretScrub(), server.WithTrustedProxies(trusted)) + // ReadTimeout bounds how long the request headers and body may take to arrive. + // It does not limit how long a handler runs, so an event stream stays open. + // WriteTimeout is left unset: a write deadline would cut that stream. httpSrv := &http.Server{ Addr: *addr, Handler: srv.Handler(), ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Minute, + IdleTimeout: 2 * time.Minute, } go func() { @@ -127,9 +132,21 @@ func run() int { <-ctx.Done() log.Println("shutting down...") - shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + drainCtx, drainCancel := context.WithTimeout(context.Background(), 15*time.Second) + drainErr := srv.Wait(drainCtx) + drainCancel() + if drainErr != nil { + log.Printf("task drain: %v", drainErr) + } + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - _ = httpSrv.Shutdown(shutdownCtx) + if err := httpSrv.Shutdown(shutdownCtx); err != nil { + log.Printf("http shutdown: %v", err) + return 1 + } + if drainErr != nil { + return 1 + } return 0 } diff --git a/db/activity_outbox.go b/db/activity_outbox.go new file mode 100644 index 00000000..c00c7751 --- /dev/null +++ b/db/activity_outbox.go @@ -0,0 +1,90 @@ +package db + +import ( + "encoding/json" + "fmt" +) + +// activityOutboxPayload is Activity plus Detail. Activity.Detail is omitted +// from ordinary JSON (the list payload stays small); the outbox has to keep it +// so a replayed row is the same step that failed to store. +type activityOutboxPayload struct { + Activity + Detail string `json:"detail,omitempty"` +} + +func marshalActivity(a Activity) ([]byte, error) { + return json.Marshal(activityOutboxPayload{Activity: a, Detail: a.Detail}) +} + +func unmarshalActivity(raw []byte) (Activity, error) { + var payload activityOutboxPayload + if err := json.Unmarshal(raw, &payload); err != nil { + return Activity{}, err + } + a := payload.Activity + a.Detail = payload.Detail + return a, nil +} + +// EnqueueActivity keeps one activity that AppendActivity could not store. +func (s *ExplorationStore) EnqueueActivity(a Activity, cause string) error { + if s == nil || s.db == nil { + return fmt.Errorf("activity outbox: no store") + } + raw, err := marshalActivity(a) + if err != nil { + return err + } + _, err = s.db.Exec(`INSERT INTO activity_outbox (exploration_id, payload, last_error) VALUES ($1, $2::jsonb, $3)`, s.expID, string(raw), cause) + return err +} + +// FlushActivityOutbox inserts queued activities in order and deletes each row +// that lands. A row that still cannot be inserted stays, with last_error +// updated, and the flush stops so later rows keep their order. A payload that +// cannot be decoded is discarded. +func (s *ExplorationStore) FlushActivityOutbox() (int, error) { + if s == nil || s.db == nil { + return 0, fmt.Errorf("activity outbox: no store") + } + rows, err := s.db.Query(`SELECT id, payload FROM activity_outbox WHERE exploration_id=$1 ORDER BY id`, s.expID) + if err != nil { + return 0, err + } + defer rows.Close() + type item struct { + id int64 + payload []byte + } + var items []item + for rows.Next() { + var it item + if err := rows.Scan(&it.id, &it.payload); err != nil { + return 0, err + } + items = append(items, it) + } + if err := rows.Err(); err != nil { + return 0, err + } + stored := 0 + for _, it := range items { + a, err := unmarshalActivity(it.payload) + if err != nil { + if _, delErr := s.db.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); delErr != nil { + return stored, delErr + } + continue + } + if _, err := s.AppendActivity(a); err != nil { + _, _ = s.db.Exec(`UPDATE activity_outbox SET last_error=$2 WHERE id=$1`, it.id, err.Error()) + return stored, err + } + if _, err := s.db.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); err != nil { + return stored, err + } + stored++ + } + return stored, nil +} diff --git a/db/activity_outbox_test.go b/db/activity_outbox_test.go new file mode 100644 index 00000000..165f3a34 --- /dev/null +++ b/db/activity_outbox_test.go @@ -0,0 +1,25 @@ +package db + +import "testing" + +func TestActivityOutboxKeepsDetail(t *testing.T) { + in := Activity{Worker: "worker", Kind: "tool_result", Summary: "done", Detail: "full body", IsError: true} + raw, err := marshalActivity(in) + if err != nil { + t.Fatal(err) + } + out, err := unmarshalActivity(raw) + if err != nil { + t.Fatal(err) + } + if out.Detail != in.Detail || out.Summary != in.Summary || out.Worker != in.Worker || out.Kind != in.Kind || !out.IsError { + t.Fatalf("round trip = %+v", out) + } +} + +func TestCoverageEnabledWithoutTask(t *testing.T) { + on, err := (&AssetStore{}).CoverageEnabled(0) + if err != nil || !on { + t.Fatalf("enabled=%v err=%v", on, err) + } +} diff --git a/db/exploration.go b/db/exploration.go index ea8a55b0..e55424b2 100644 --- a/db/exploration.go +++ b/db/exploration.go @@ -354,7 +354,10 @@ WHERE exploration_id=$1 AND kind='intent' AND state='running' if err != nil { return 0, err } - n, _ := res.RowsAffected() + n, err := res.RowsAffected() + if err != nil { + return 0, err + } return n, nil } diff --git a/db/llm_usage.go b/db/llm_usage.go index a721aa4f..8d8884d7 100644 --- a/db/llm_usage.go +++ b/db/llm_usage.go @@ -33,6 +33,17 @@ func (d *DB) EnsureLLMUsageTable() error { return d.execEmbeddedMigration(llmUsageMigration) } +// TaskLLMSpend is the token and turn total already recorded for a task. +// One row is one turn. Cache read and write are not added: input_tokens already +// includes them. Every status counts, because a failed call is metered too. +func (d *DB) TaskLLMSpend(taskID string) (tokens, turns int64, err error) { + if d == nil || taskID == "" { + return 0, 0, nil + } + err = d.QueryRow(`SELECT COALESCE(SUM(input_tokens + output_tokens), 0), COUNT(*) FROM llm_usage WHERE task_id=$1`, taskID).Scan(&tokens, &turns) + return tokens, turns, err +} + // InsertLLMUsage appends one metering row. Best-effort: callers log and continue on // error (a lost metering row must never break the LLM call). func (d *DB) InsertLLMUsage(u *LLMUsage) error { diff --git a/db/migrate_test.go b/db/migrate_test.go index f582fd83..2f914317 100644 --- a/db/migrate_test.go +++ b/db/migrate_test.go @@ -136,9 +136,9 @@ func assertEmbeddedApplied(t *testing.T, d *DB) []appliedMigration { want := []struct { version int name string - }{{1, llmRecordsMigration}, {2, llmUsageMigration}, {3, "0003_task_scope_icp_index.sql"}, {4, "0004_llm_usage_ts_index.sql"}} + }{{1, llmRecordsMigration}, {2, llmUsageMigration}, {3, "0003_task_scope_icp_index.sql"}, {4, "0004_llm_usage_ts_index.sql"}, {5, "0005_activity_outbox.sql"}} if len(applied) != len(want) { - t.Fatalf("schema_migrations = %+v, want versions 1 to 4", applied) + t.Fatalf("schema_migrations = %+v, want versions 1 to 5", applied) } for i, w := range want { content, err := fs.ReadFile(migrationsFS(), w.name) diff --git a/db/migrations/0005_activity_outbox.sql b/db/migrations/0005_activity_outbox.sql new file mode 100644 index 00000000..ff1396c4 --- /dev/null +++ b/db/migrations/0005_activity_outbox.sql @@ -0,0 +1,14 @@ +-- Activities that could not be inserted (the exploration row was missing, or +-- the write failed after retries) are kept here and replayed into activity +-- once the parent row accepts them. No foreign key: the row has to survive +-- the missing exploration that caused the original failure. +CREATE TABLE IF NOT EXISTS activity_outbox ( + id BIGSERIAL PRIMARY KEY, + exploration_id BIGINT NOT NULL, + payload JSONB NOT NULL, + last_error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS activity_outbox_exploration_idx + ON activity_outbox (exploration_id, id); diff --git a/db/task_scope.go b/db/task_scope.go index bc9defe0..bbbdee84 100644 --- a/db/task_scope.go +++ b/db/task_scope.go @@ -297,17 +297,22 @@ type Coverage struct { } // CoverageEnabled reports whether a task has the asset-coverage feature turned on -// (tasks.coverage_enabled). Missing row / error → true (fail open to the default), -// so unknown/legacy tasks keep the historical behavior. taskID<=0 → true. -func (s *AssetStore) CoverageEnabled(taskID int64) bool { +// (tasks.coverage_enabled). A missing row stays enabled, which is the historical +// default for unknown and legacy tasks. taskID<=0 stays enabled. A query error is +// returned with enabled=true so a database blip does not turn the feature off. +func (s *AssetStore) CoverageEnabled(taskID int64) (bool, error) { if taskID <= 0 { - return true + return true, nil } var enabled bool - if err := s.db.QueryRow(`SELECT COALESCE(coverage_enabled,true) FROM tasks WHERE id=$1`, taskID).Scan(&enabled); err != nil { - return true + err := s.db.QueryRow(`SELECT COALESCE(coverage_enabled,true) FROM tasks WHERE id=$1`, taskID).Scan(&enabled) + if err == sql.ErrNoRows { + return true, nil } - return enabled + if err != nil { + return true, err + } + return enabled, nil } // declaredScopeFilter keeps a task_scope row ts in the coverage match only when it is diff --git a/server/auth.go b/server/auth.go index b76a1d2d..28ba185b 100644 --- a/server/auth.go +++ b/server/auth.go @@ -19,13 +19,15 @@ import ( "golang.org/x/crypto/bcrypt" "github.com/Autumn-27/artex/config" + "github.com/Autumn-27/artex/db" ) const ( - jwtKeyFilename = "jwt.key" - authPassKey = "auth.password_hash" - jwtTTL = 7 * 24 * time.Hour - keyChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + jwtKeyFilename = "jwt.key" + authPassKey = "auth.password_hash" + authPwdEpochKey = "auth.password_epoch" + jwtTTL = 7 * 24 * time.Hour + keyChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" // The lower bound matches the setup page's frontend validation — validation that // lives only in the frontend is as good as none: hitting the API directly bypasses @@ -167,15 +169,24 @@ func signJWT(key []byte) (string, error) { }).SignedString(key) } -// verifyJWT returns true when tokenStr is a valid, non-expired HS256 token. -func verifyJWT(tokenStr string, key []byte) bool { - t, err := jwt.Parse(tokenStr, func(t *jwt.Token) (any, error) { +// verifyJWT returns true when tokenStr is a valid, non-expired HS256 token +// issued at or after notBefore. notBefore is Unix seconds; 0 means no cutoff, +// so a deployment that has never stored a password epoch keeps existing sessions. +func verifyJWT(tokenStr string, key []byte, notBefore int64) bool { + claims := &jwt.RegisteredClaims{} + t, err := jwt.ParseWithClaims(tokenStr, claims, func(t *jwt.Token) (any, error) { if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok { return nil, fmt.Errorf("unexpected signing method") } return key, nil }) - return err == nil && t.Valid + if err != nil || !t.Valid { + return false + } + if notBefore > 0 && (claims.IssuedAt == nil || claims.IssuedAt.Unix() < notBefore) { + return false + } + return true } // jwtSubject returns the subject of tokenStr when it is a valid, non-expired @@ -197,7 +208,11 @@ func jwtSubject(tokenStr string, key []byte) string { // approver is who a human decision is recorded for: the subject of the // request's session token. func (s *Server) approver(r *http.Request) string { - return jwtSubject(extractToken(r), s.jwtKey) + tok := extractToken(r) + if !verifyJWT(tok, s.jwtKey, s.pwdEpoch.Load()) { + return "" + } + return jwtSubject(tok, s.jwtKey) } // extractToken reads the JWT from Authorization: Bearer header, @@ -226,7 +241,7 @@ func (s *Server) requireAuth(h http.Handler) http.Handler { writeErr(w, 401, "Unauthorized") return } - if !verifyJWT(tok, s.jwtKey) { + if !verifyJWT(tok, s.jwtKey, s.pwdEpoch.Load()) { writeErr(w, 401, "token is invalid or expired") return } @@ -258,15 +273,72 @@ func (s *Server) initSetupToken() { if s.m == nil || s.m.pg == nil { return } - if hash, _, err := s.m.pg.GetSetting(authPassKey); err == nil && hash != "" { + var hash string + var err error + for attempt := 1; attempt <= 3; attempt++ { + var ok bool + hash, ok, err = s.m.pg.GetSetting(authPassKey) + if err == nil { + if ok && hash != "" { + return + } + break + } + log.Printf("[auth] password lookup attempt %d/3: %v", attempt, err) + if attempt < 3 { + time.Sleep(time.Duration(attempt) * 100 * time.Millisecond) + } + } + if err != nil { + log.Printf("[auth] password lookup failed; not issuing a setup token: %v", err) return } buf := make([]byte, 16) if _, err := rand.Read(buf); err != nil { log.Fatalf("[auth] generate setup token: %v", err) } - s.setupToken = hex.EncodeToString(buf) - log.Printf("[auth] no admin password is set. Setup token for the /setup page: %s", s.setupToken) + token := hex.EncodeToString(buf) + s.setupMu.Lock() + s.setupToken = token + s.setupMu.Unlock() + log.Printf("[auth] no admin password is set. Setup token for the /setup page: %s", token) +} + +// loadPwdEpoch reads auth.password_epoch. A missing key leaves the cutoff at +// zero, which accepts every valid token. +func (s *Server) loadPwdEpoch() { + if s.m == nil || s.m.pg == nil { + return + } + raw, ok, err := s.m.pg.GetSetting(authPwdEpochKey) + if err != nil { + log.Printf("[auth] password epoch: %v", err) + return + } + if !ok || raw == "" { + return + } + n, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + log.Printf("[auth] password epoch %q: %v", raw, err) + return + } + s.pwdEpoch.Store(n) +} + +// bumpPwdEpoch records the current second as the oldest token that is still +// valid. A token signed in that same second still passes. A failed write leaves +// the previous cutoff in place and is logged; the password change itself stands. +func (s *Server) bumpPwdEpoch(pg *db.DB) { + if pg == nil { + return + } + epoch := time.Now().Unix() + if err := pg.SetSetting(authPwdEpochKey, strconv.FormatInt(epoch, 10)); err != nil { + log.Printf("[auth] save password epoch: %v", err) + return + } + s.pwdEpoch.Store(epoch) } // checkSetupToken compares got with the active setup token in constant time. @@ -330,6 +402,7 @@ func (s *Server) authInit(w http.ResponseWriter, r *http.Request) { s.setupMu.Lock() s.setupToken = "" s.setupMu.Unlock() + s.bumpPwdEpoch(pg) tok, err := signJWT(s.jwtKey) if err != nil { writeErr(w, 500, "token generation failed") @@ -346,7 +419,7 @@ func (s *Server) authChangePassword(w http.ResponseWriter, r *http.Request) { if pg == nil { return } - if !verifyJWT(extractToken(r), s.jwtKey) { + if !verifyJWT(extractToken(r), s.jwtKey, s.pwdEpoch.Load()) { writeErr(w, 401, "Unauthorized") return } @@ -388,6 +461,9 @@ func (s *Server) authChangePassword(w http.ResponseWriter, r *http.Request) { writeErr(w, 500, "Save failed: "+err.Error()) return } + // Sessions issued before this second stop working. This response does not + // include a replacement token; the next request asks for a new login. + s.bumpPwdEpoch(pg) writeJSON(w, 200, map[string]any{"ok": true}) } diff --git a/server/engine.go b/server/engine.go index eef8645f..78f62b67 100644 --- a/server/engine.go +++ b/server/engine.go @@ -118,6 +118,7 @@ type Engine struct { // runs under one task-scoped context. Successful deletion cancels that context, // waits for all goroutines, then releases every task-level Engine reference. runtimeMu sync.Mutex + draining bool runtimes map[string]*taskRuntime // per-task execution context: each planner.Plan / worker.Execute runs under it, @@ -231,6 +232,9 @@ func (e *Engine) IsDeleting(taskID string) bool { func (e *Engine) registerTaskRoutines(parent context.Context, taskID string, count int) *taskRuntime { e.runtimeMu.Lock() defer e.runtimeMu.Unlock() + if e.draining { + return nil + } rt := e.runtimes[taskID] if rt == nil { ctx, cancel := context.WithCancel(parent) @@ -248,6 +252,42 @@ func runTaskRoutine(rt *taskRuntime, fn func(context.Context)) { }() } +// Wait blocks until the task goroutines registered at the moment of the call +// have finished, or until ctx ends. It refuses new registrations first so Add +// cannot race Wait. Goroutines that were already running are cancelled with +// their task context. +func (e *Engine) Wait(ctx context.Context) error { + if e == nil { + return nil + } + e.runtimeMu.Lock() + e.draining = true + groups := make([]*sync.WaitGroup, 0, len(e.runtimes)) + for _, rt := range e.runtimes { + if rt == nil { + continue + } + if rt.cancel != nil { + rt.cancel() + } + groups = append(groups, &rt.wg) + } + e.runtimeMu.Unlock() + for _, wg := range groups { + done := make(chan struct{}) + go func(wg *sync.WaitGroup) { + wg.Wait() + close(done) + }(wg) + select { + case <-done: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil +} + // StopTask permanently stops every long-lived goroutine and removes all Engine // state for a successfully deleted task. The delete barrier remains installed // until cleanup finishes, so no new task operation can race the teardown. @@ -664,6 +704,23 @@ func (e *Engine) emitActivity(t *Task, r db.Activity) db.Activity { } log.Printf("[activity] task %s dropping activity record (%d dropped for this task so far) worker=%s kind=%s tool=%s tuid=%s reason=%s summary=%q: %v%s", t.ID, n, r.Worker, r.Kind, r.Tool, r.ToolUseID, dropReason(err), preview(r.Summary, 80), err, diag) + if t.Store != nil { + if qerr := t.Store.EnqueueActivity(r, err.Error()); qerr != nil { + log.Printf("[activity] task %s outbox: %v", t.ID, qerr) + } + } + // The row has no database id. Publish an error copy so a live session + // does not stay on "Running" while the original return value is unchanged. + notice := r + notice.IsError = true + if notice.Summary == "" { + notice.Summary = "activity was not stored" + } else { + notice.Summary += " (not stored)" + } + if e.bc != nil { + e.bc.Publish(t.ID, notice) + } e.touch(t.ID) return r } @@ -762,6 +819,10 @@ func (e *Engine) Run(ctx context.Context, t *Task) { } rt := e.registerTaskRoutines(ctx, t.ID, 2+workers) e.deleteMu.RUnlock() + if rt == nil { + e.started.Delete(t.ID) + return + } e.touch(t.ID) runTaskRoutine(rt, func(loopCtx context.Context) { e.plannerLoop(loopCtx, t) }) runTaskRoutine(rt, func(loopCtx context.Context) { e.claimSweepLoop(loopCtx, t) }) diff --git a/server/engine_budget.go b/server/engine_budget.go index b6021b2d..a11ab7a5 100644 --- a/server/engine_budget.go +++ b/server/engine_budget.go @@ -17,8 +17,33 @@ func (e *Engine) budgetFor(taskID string) *agent.BudgetLedger { if v, ok := e.budgets.Load(taskID); ok { return v.(*agent.BudgetLedger) } - v, _ := e.budgets.LoadOrStore(taskID, agent.NewBudgetLedger(agent.BudgetsFromEnv())) - return v.(*agent.BudgetLedger) + created := agent.NewBudgetLedger(agent.BudgetsFromEnv()) + e.seedBudget(taskID, created) + v, loaded := e.budgets.LoadOrStore(taskID, created) + if loaded { + return v.(*agent.BudgetLedger) + } + return created +} + +// seedBudget rebuilds spent tokens and turns from llm_usage the first time a +// ledger is created in this process. The in-memory sink runs before the row is +// inserted, so a call that happens after this seed is counted once. A nil +// ledger means no budget is configured and nothing is read. +func (e *Engine) seedBudget(taskID string, ledger *agent.BudgetLedger) { + if ledger == nil || e == nil || e.m == nil || e.m.pg == nil || taskID == "" { + return + } + tokens, turns, err := e.m.pg.TaskLLMSpend(taskID) + if err != nil { + log.Printf("[budget] task %s: cannot restore spend from llm_usage: %v", taskID, err) + return + } + if tokens == 0 && turns == 0 { + return + } + ledger.RestoreSpend(agent.Spend{Tokens: tokens, Turns: turns}) + log.Printf("[budget] task %s restored %d tokens / %d turns from llm_usage", taskID, tokens, turns) } // budgetStop is the enforcement point of a planner round: it logs the remaining diff --git a/server/engine_timeout.go b/server/engine_timeout.go index 37f5469f..cb596d02 100644 --- a/server/engine_timeout.go +++ b/server/engine_timeout.go @@ -161,6 +161,10 @@ func (e *Engine) startDeadlineCoordinator(ctx context.Context, t *Task) { } rt := e.registerTaskRoutines(ctx, t.ID, 1) e.deleteMu.RUnlock() + if rt == nil { + e.coordStarted.Delete(t.ID) + return + } runTaskRoutine(rt, func(loopCtx context.Context) { e.deadlineCoordinator(loopCtx, t) }) } diff --git a/server/llmpool.go b/server/llmpool.go index f892d344..85519f59 100644 --- a/server/llmpool.go +++ b/server/llmpool.go @@ -2,6 +2,7 @@ package server import ( "log" + "sync" "time" "github.com/Autumn-27/artex/agent" @@ -23,20 +24,12 @@ func newLLMHealthRegistry(pg *db.DB) *llmpool.Registry { if pg == nil { return llmpool.NewRegistry(nil, nil) } + syncer := newHealthSync(pg.SaveLLMHealth, pg.ClearLLMHealth) persist := func(id int64, st llmpool.State) { - h := db.LLMHealth{ProfileID: id, Fails: st.Fails, Trips: st.Trips, LastError: st.LastError} - if !st.OpenUntil.IsZero() { - t := st.OpenUntil - h.OpenUntil = &t - } - go func() { - if err := pg.SaveLLMHealth(h); err != nil { - log.Printf("[llmpool] failed to persist circuit-breaker state: %v", err) - } - }() + syncer.enqueue(id, healthOp{st: st}) } forget := func(id int64) { - go func() { _ = pg.ClearLLMHealth(id) }() + syncer.enqueue(id, healthOp{clear: true}) } reg := llmpool.NewRegistry(persist, forget) // Restore only windows that haven't expired (LoadLLMHealth filters), so a @@ -54,6 +47,93 @@ func newLLMHealthRegistry(pg *db.DB) *llmpool.Registry { return reg } +// healthOp is the latest circuit-breaker write for one profile. clear drops the row. +type healthOp struct { + clear bool + st llmpool.State +} + +// healthSync writes circuit-breaker state from one goroutine. A newer update for +// the same profile replaces one that has not been written yet, so a burst of +// state changes becomes one database call and the request path does not wait. +type healthSync struct { + save func(db.LLMHealth) error + clear func(int64) error + mu sync.Mutex + pending map[int64]healthOp + wake chan struct{} +} + +func newHealthSync(save func(db.LLMHealth) error, clear func(int64) error) *healthSync { + h := &healthSync{ + save: save, + clear: clear, + pending: map[int64]healthOp{}, + wake: make(chan struct{}, 1), + } + go h.loop() + return h +} + +func (h *healthSync) enqueue(id int64, op healthOp) { + h.mu.Lock() + if h.pending == nil { + h.pending = map[int64]healthOp{} + } + h.pending[id] = op + h.mu.Unlock() + if h.wake == nil { + return + } + select { + case h.wake <- struct{}{}: + default: + } +} + +func (h *healthSync) take() map[int64]healthOp { + h.mu.Lock() + defer h.mu.Unlock() + if len(h.pending) == 0 { + return nil + } + out := h.pending + h.pending = map[int64]healthOp{} + return out +} + +func (h *healthSync) loop() { + for { + batch := h.take() + if len(batch) == 0 { + <-h.wake + continue + } + for id, op := range batch { + if op.clear { + if h.clear == nil { + continue + } + if err := h.clear(id); err != nil { + log.Printf("[llmpool] clear circuit-breaker profile #%d: %v", id, err) + } + continue + } + if h.save == nil { + continue + } + row := db.LLMHealth{ProfileID: id, Fails: op.st.Fails, Trips: op.st.Trips, LastError: op.st.LastError} + if !op.st.OpenUntil.IsZero() { + openUntil := op.st.OpenUntil + row.OpenUntil = &openUntil + } + if err := h.save(row); err != nil { + log.Printf("[llmpool] persist circuit-breaker profile #%d: %v", id, err) + } + } + } +} + // poolMember builds one chain member from a profile, reusing the per-profile // provider cache so every agent pointing at the same profile shares one provider // (hence one rate limiter). Returns nil when the profile can't be built. diff --git a/server/login_limit.go b/server/login_limit.go index 51dec5af..fcc0d1e6 100644 --- a/server/login_limit.go +++ b/server/login_limit.go @@ -105,16 +105,34 @@ func (l *loginLimiter) sweep(now time.Time) { } } -// evictOldest drops the least recently seen entry to stay under loginMaxEntries. +// evictOldest drops the least recently seen entry that is not inside a block. +// A block that is still running stays, so filling the map cannot clear an +// active lockout. When every entry is blocked, the one whose block ends soonest +// is the one that goes. func (l *loginLimiter) evictOldest() { + now := l.now() var oldest string var at time.Time + found := false for ip, e := range l.entries { - if oldest == "" || e.lastSeen.Before(at) { - oldest, at = ip, e.lastSeen + if now.Before(e.blockedUntil) { + continue + } + if !found || e.lastSeen.Before(at) { + oldest, at, found = ip, e.lastSeen, true } } - delete(l.entries, oldest) + if !found { + var until time.Time + for ip, e := range l.entries { + if oldest == "" || e.blockedUntil.Before(until) { + oldest, until = ip, e.blockedUntil + } + } + } + if oldest != "" { + delete(l.entries, oldest) + } } // peerIP is the host part of the TCP peer address. diff --git a/server/login_limit_test.go b/server/login_limit_test.go index 255b64c8..d0a033a9 100644 --- a/server/login_limit_test.go +++ b/server/login_limit_test.go @@ -274,6 +274,36 @@ func TestLoginLimiterCapsEntries(t *testing.T) { } } +func TestEvictOldestKeepsBlockedEntries(t *testing.T) { + clk := newFakeClock() + l := newLoginLimiter(clk.now) + now := clk.now() + l.entries["blocked"] = &loginEntry{lastSeen: now.Add(-2 * time.Hour), blockedUntil: now.Add(time.Hour), block: time.Minute} + l.entries["idle"] = &loginEntry{lastSeen: now.Add(-time.Hour)} + l.evictOldest() + if _, ok := l.entries["blocked"]; !ok { + t.Fatal("evicted a blocked address while an idle one existed") + } + if _, ok := l.entries["idle"]; ok { + t.Fatal("kept the idle address") + } +} + +func TestEvictOldestDropsSoonestBlockWhenAllBlocked(t *testing.T) { + clk := newFakeClock() + l := newLoginLimiter(clk.now) + now := clk.now() + l.entries["long"] = &loginEntry{lastSeen: now, blockedUntil: now.Add(2 * time.Hour)} + l.entries["short"] = &loginEntry{lastSeen: now.Add(-time.Hour), blockedUntil: now.Add(time.Minute)} + l.evictOldest() + if _, ok := l.entries["short"]; ok { + t.Fatal("kept the block that ends sooner") + } + if _, ok := l.entries["long"]; !ok { + t.Fatal("evicted the longer block") + } +} + func TestHealthHidesVersionWithoutToken(t *testing.T) { key := []byte("0123456789abcdef0123456789abcdef") s := &Server{jwtKey: key} diff --git a/server/reliability_test.go b/server/reliability_test.go new file mode 100644 index 00000000..447c7653 --- /dev/null +++ b/server/reliability_test.go @@ -0,0 +1,89 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/artex/llmpool" +) + +func TestDecodeAcceptsSmallBody(t *testing.T) { + r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"x":"ok"}`)) + var v map[string]string + if err := decode(r, &v); err != nil || v["x"] != "ok" { + t.Fatalf("decode: %v %+v", err, v) + } +} + +func TestDecodeCapsBody(t *testing.T) { + body := strings.NewReader(`{"x":"` + strings.Repeat("a", decodeMaxBytes) + `"}`) + r := httptest.NewRequest(http.MethodPost, "/", body) + var v map[string]string + if err := decode(r, &v); err == nil { + t.Fatal("body over the limit was accepted") + } +} + +func TestTokenBeforePasswordEpochIsRejected(t *testing.T) { + key := []byte("0123456789abcdef0123456789abcdef") + tok, err := signJWT(key) + if err != nil { + t.Fatal(err) + } + if !verifyJWT(tok, key, 0) { + t.Fatal("a token with no password epoch must be accepted") + } + if verifyJWT(tok, key, time.Now().Add(time.Minute).Unix()) { + t.Fatal("a token issued before the password epoch must be rejected") + } + if !verifyJWT(tok, key, time.Now().Add(-time.Minute).Unix()) { + t.Fatal("a token issued after the password epoch must be accepted") + } +} + +func TestHealthSyncCoalescesToLatest(t *testing.T) { + h := &healthSync{pending: map[int64]healthOp{}} + h.enqueue(7, healthOp{st: llmpool.State{Fails: 1}}) + h.enqueue(7, healthOp{st: llmpool.State{Fails: 4}}) + h.enqueue(7, healthOp{clear: true}) + h.enqueue(8, healthOp{st: llmpool.State{Fails: 2}}) + batch := h.take() + if len(batch) != 2 || !batch[7].clear || batch[8].st.Fails != 2 { + t.Fatalf("batch=%+v", batch) + } + if again := h.take(); again != nil { + t.Fatalf("pending was not cleared: %+v", again) + } + var saved []db.LLMHealth + var cleared []int64 + worker := &healthSync{ + save: func(row db.LLMHealth) error { + saved = append(saved, row) + return nil + }, + clear: func(id int64) error { + cleared = append(cleared, id) + return nil + }, + } + worker.enqueue(7, healthOp{clear: true}) + worker.enqueue(8, healthOp{st: llmpool.State{Fails: 3, LastError: "down"}}) + for id, op := range worker.take() { + if op.clear { + if err := worker.clear(id); err != nil { + t.Fatal(err) + } + continue + } + if err := worker.save(db.LLMHealth{ProfileID: id, Fails: op.st.Fails, LastError: op.st.LastError}); err != nil { + t.Fatal(err) + } + } + if len(cleared) != 1 || cleared[0] != 7 || len(saved) != 1 || saved[0].ProfileID != 8 || saved[0].Fails != 3 { + t.Fatalf("cleared=%v saved=%+v", cleared, saved) + } +} diff --git a/server/server.go b/server/server.go index 56e74734..c83b69b0 100644 --- a/server/server.go +++ b/server/server.go @@ -18,6 +18,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/Autumn-27/artex/agent" @@ -52,6 +53,10 @@ type Server struct { setupMu sync.Mutex setupToken string // one-time token POST /api/auth/init requires; empty when none is active + // pwdEpoch is auth.password_epoch in Unix seconds. Zero means the setting + // is absent and every valid token is accepted. A token issued before the + // stored second is rejected. + pwdEpoch atomic.Int64 loginLimit *loginLimiter // per-IP attempt limit on POST /api/auth/login trustedProxies []netip.Prefix // ARTEX_TRUSTED_PROXIES; see clientIP @@ -169,6 +174,7 @@ func New(ctx context.Context, m *Manager, skillDir string, dataDir string, keyDi o(s) } s.initSetupToken() + s.loadPwdEpoch() s.initSideQuestions() // Circuit-breaker thresholds/cooldowns are hot parameters on the failure path; push the global retry policy to the Registry once at startup, // and again on every policy save (saveLLMRetryPolicy). @@ -295,9 +301,16 @@ func (s *Server) restoreTaskRuntimes() { lifecycle := t.lifecycleSnapshot() // clear stale 'running' intents from a prior crash/restart (no live worker // owns them) so they re-claim instead of spinning forever in the UI. - if n, _ := t.Store.ResetRunningIntents(); n > 0 { + if n, err := t.Store.ResetRunningIntents(); err != nil { + log.Printf("[engine] task %s reset leftover running intents: %v", t.ID, err) + } else if n > 0 { log.Printf("[engine] task %s reset %d leftover running intents to open", t.ID, n) } + if n, err := t.Store.FlushActivityOutbox(); err != nil { + log.Printf("[activity] task %s outbox: %v", t.ID, err) + } else if n > 0 { + log.Printf("[activity] task %s stored %d activities recovered from the outbox", t.ID, n) + } if lifecycle.Paused { s.engine.Pause(t.ID, agent.AbortPausedOnReload) } @@ -319,6 +332,30 @@ func (s *Server) restoreTaskRuntimes() { } } +// Wait blocks until the task goroutines snapshotted now, and the archive +// worker, have finished, or until ctx ends. +func (s *Server) Wait(ctx context.Context) error { + if s == nil { + return nil + } + if s.engine != nil { + if err := s.engine.Wait(ctx); err != nil { + return err + } + } + done := make(chan struct{}) + go func() { + s.archiveWG.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + // agentMaxTurns returns the configured max_turns for an agent key (0 = unlimited, // also the fallback when no DB or no row). func (s *Server) agentMaxTurns(key string) int { @@ -991,7 +1028,7 @@ func (s *Server) Handler() http.Handler { // with a valid token gets the service name and version; the web UI's top bar // reads the version, and sends its token. func (s *Server) health(w http.ResponseWriter, r *http.Request) { - if !verifyJWT(extractToken(r), s.jwtKey) { + if !verifyJWT(extractToken(r), s.jwtKey, s.pwdEpoch.Load()) { writeJSON(w, 200, map[string]any{"ok": true}) return } diff --git a/server/server_mgmt.go b/server/server_mgmt.go index c51e9fb1..903f1a2d 100644 --- a/server/server_mgmt.go +++ b/server/server_mgmt.go @@ -79,7 +79,15 @@ func pathInt(r *http.Request, name string) (int64, bool) { return n, err == nil } -func decode(r *http.Request, v any) error { return json.NewDecoder(r.Body).Decode(v) } +// decodeMaxBytes is the default JSON body cap. Handlers that already wrapped +// the body with a smaller MaxBytesReader keep that smaller cap. Multipart +// uploads do not use decode. +const decodeMaxBytes = 32 << 20 + +func decode(r *http.Request, v any) error { + r.Body = http.MaxBytesReader(nil, r.Body, decodeMaxBytes) + return json.NewDecoder(r.Body).Decode(v) +} // ---------- tasks (delete) ---------- diff --git a/traffic/passthrough_test.go b/traffic/passthrough_test.go index 26f0cabb..d9227a3b 100644 --- a/traffic/passthrough_test.go +++ b/traffic/passthrough_test.go @@ -4,6 +4,7 @@ import ( "errors" "net/url" "testing" + "time" mproxy "github.com/lqqyt2423/go-mitmproxy/proxy" ) @@ -65,4 +66,23 @@ func TestMaybePassthroughFlagsHostOnce(t *testing.T) { if _, tunnel := tr.pass.Load(hostOnly("target.test:443")); !tunnel { t.Fatal("flagged host must be recognized for the CONNECT form with port") } + if !tr.tunneling("target.test") { + t.Fatal("a fresh passthrough mark must tunnel") + } +} + +func TestPassthroughMarkExpires(t *testing.T) { + tr := &Traffic{} + host := "expired.example" + tr.pass.Store(host, time.Now().Add(-time.Second)) + if tr.tunneling(host) { + t.Fatal("expired passthrough mark still tunnels") + } + if _, ok := tr.pass.Load(host); ok { + t.Fatal("expired mark was kept") + } + tr.pass.Store(host, time.Now().Add(time.Minute)) + if !tr.tunneling(host) { + t.Fatal("fresh passthrough mark should tunnel") + } } diff --git a/traffic/traffic.go b/traffic/traffic.go index 6ac3709f..fb0dfdba 100644 --- a/traffic/traffic.go +++ b/traffic/traffic.go @@ -163,8 +163,9 @@ type Traffic struct { closeOnce sync.Once // pass is the set of hosts whose MITM interception failed for a proxy/protocol // reason; connections to them are tunneled transparently (fail-open) so the - // request still reaches the target — unrecorded — instead of being killed. - pass sync.Map // hostname(string) -> struct{} + // request still reaches the target, unrecorded, instead of being killed. + // The value is the time the mark expires. After that, interception is tried again. + pass sync.Map // hostname(string) -> time.Time // upstream is the global egress proxy every captured request is forwarded // through (nil = dial targets directly). Both the intercepted and the // transparent-passthrough paths honor it (go-mitmproxy's getUpstreamConn), @@ -223,8 +224,7 @@ func Open(dir, addr string) (*Traffic, error) { // can't intercept without breaking (see maybePassthrough). Those are tunneled // transparently so the request still reaches the target instead of being killed. p.SetShouldInterceptRule(func(req *http.Request) bool { - _, tunnel := t.pass.Load(hostOnly(req.Host)) - return !tunnel + return !t.tunneling(hostOnly(req.Host)) }) p.AddAddon(&sink{t: t}) t.proxy = p @@ -415,9 +415,30 @@ func (t *Traffic) maybePassthrough(f *mproxy.Flow, err error) { if host == "" { return } - if _, loaded := t.pass.LoadOrStore(host, struct{}{}); !loaded { - log.Printf("[traffic] MITM with %s failed, falling back to passthrough (this host now connects straight to the target and is no longer recorded, but requests proceed normally): %v", host, err) + if t.tunneling(host) { + return + } + until := time.Now().Add(passthroughTTL) + if _, loaded := t.pass.LoadOrStore(host, until); !loaded { + log.Printf("[traffic] MITM with %s failed, falling back to passthrough for %s (this host connects straight to the target and is not recorded until then; requests still proceed): %v", host, passthroughTTL, err) + } +} + +const passthroughTTL = time.Minute + +// tunneling reports whether host is inside an unexpired passthrough mark. +// An expired mark is removed so the next connection tries interception again. +func (t *Traffic) tunneling(host string) bool { + v, ok := t.pass.Load(host) + if !ok { + return false + } + until, ok := v.(time.Time) + if !ok || !time.Now().Before(until) { + t.pass.CompareAndDelete(host, v) + return false } + return true } // proxyCausedErr reports whether err indicates the interception layer (not the diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 03f5eb28..0e6b16cc 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -163,7 +163,9 @@ export async function http(path: string, init?: RequestInit): Promise { // to the Go backend on :8787, whose CORS is open. // // Override either default with NEXT_PUBLIC_SSE_BASE (set it to "" to force same-origin). -// Token is appended as ?token= because SSE can't carry cookies cross-origin. +// A same-origin stream sends the artex_token cookie. The query token is added only +// when base is non-empty, which is the dev connection to :8787: that origin cannot +// send the cookie. // mockReport returns a canned Markdown report for the demo. function mockReport(_task?: string): string { return `# ARTEX Penetration Test Report — Acme Corp @@ -193,6 +195,9 @@ export function sseUrl(path: string): string { (process.env.NODE_ENV !== "production" && typeof window !== "undefined" ? `${window.location.protocol}//${window.location.hostname}:8787` : ""); + if (!base) { + return path; + } const token = getToken(); const sep = path.includes("?") ? "&" : "?"; return token ? `${base}${path}${sep}token=${encodeURIComponent(token)}` : `${base}${path}`;