From be25f1497083393c4fb91319a247e232c9b5e0b9 Mon Sep 17 00:00:00 2001 From: protonmatter Date: Tue, 17 Nov 2026 13:04:36 -0500 Subject: [PATCH 1/3] Restore a task spend cap from the lanes the live ledger counts. Planner, worker, judge, and compactor rows rebuild the cap. Main-agent chat, goal decomposition, and side questions stay out, so a restart does not count turns the running process never metered. --- agent/budget.go | 13 +++++++++++++ agent/budget_test.go | 11 +++++++++++ db/llm_usage.go | 18 ++++++++++++++++++ server/engine_budget.go | 29 +++++++++++++++++++++++++++-- 4 files changed, 69 insertions(+), 2 deletions(-) diff --git a/agent/budget.go b/agent/budget.go index f22028e02..8fe727e55 100644 --- a/agent/budget.go +++ b/agent/budget.go @@ -110,6 +110,19 @@ func NewBudgetLedger(task, intent Budget) *BudgetLedger { return &BudgetLedger{task: task, intent: intent} } +// RestoreSpend adds usage that was already recorded before this process started. +// It is applied once, to a ledger that has not recorded anything yet. A nil +// ledger (no budget configured) is a no-op. +func (l *BudgetLedger) RestoreSpend(spent Spend) { + if l == nil || (spent.Tokens == 0 && spent.Turns == 0) { + return + } + l.mu.Lock() + l.spent.Tokens += spent.Tokens + l.spent.Turns += spent.Turns + l.mu.Unlock() +} + // Record meters one task-level call (the planner's own rounds). func (l *BudgetLedger) Record(u llm.Usage) { l.mu.Lock() diff --git a/agent/budget_test.go b/agent/budget_test.go index b35f35884..2c4c10a18 100644 --- a/agent/budget_test.go +++ b/agent/budget_test.go @@ -144,3 +144,14 @@ func TestBudgetCountsMatchUsageFixture(t *testing.T) { t.Fatalf("spent %+v remaining %+v estimate %+v, want {%d 5}, remaining, {10400 3}", st.Spent, st.Remaining, st.Estimate, want) } } + +func TestRestoreSpendCountsTowardCap(t *testing.T) { + l := NewBudgetLedger(NewBudget(100, 10), NewBudget(0, 0)) + l.RestoreSpend(Spend{Tokens: 40, Turns: 4}) + st := l.State() + if st.Spent != (Spend{40, 4}) || st.Remaining != (Budget{60, 6}) { + t.Fatalf("spent=%+v remaining=%+v", st.Spent, st.Remaining) + } + var none *BudgetLedger + none.RestoreSpend(Spend{Tokens: 1, Turns: 1}) +} diff --git a/db/llm_usage.go b/db/llm_usage.go index a721aa4ff..5945974f0 100644 --- a/db/llm_usage.go +++ b/db/llm_usage.go @@ -33,6 +33,24 @@ func (d *DB) EnsureLLMUsageTable() error { return d.execEmbeddedMigration(llmUsageMigration) } +// TaskLLMSpend is the token and turn total the in-memory task ledger would +// already have counted. One row is one turn. The ledger sees planner and worker +// completions, plus judge and compactor calls made on those contexts. Main-agent +// chat, goal decomposition, and side questions write llm_usage rows for the same +// task and never enter the ledger, so they are left out: counting them would +// tighten the turn cap after a restart. Cache read and write are not added: +// input_tokens already includes them. Every status counts, because a failed call +// is metered too. +func (d *DB) TaskLLMSpend(taskID string) (tokens, turns int64, err error) { + if d == nil || taskID == "" { + return 0, 0, nil + } + err = d.QueryRow(`SELECT COALESCE(SUM(input_tokens + output_tokens), 0), COUNT(*) +FROM llm_usage +WHERE task_id=$1 AND worker IN ('planner', 'worker', 'judge', 'compactor')`, taskID).Scan(&tokens, &turns) + return tokens, turns, err +} + // InsertLLMUsage appends one metering row. Best-effort: callers log and continue on // error (a lost metering row must never break the LLM call). func (d *DB) InsertLLMUsage(u *LLMUsage) error { diff --git a/server/engine_budget.go b/server/engine_budget.go index b6021b2d6..a11ab7a59 100644 --- a/server/engine_budget.go +++ b/server/engine_budget.go @@ -17,8 +17,33 @@ func (e *Engine) budgetFor(taskID string) *agent.BudgetLedger { if v, ok := e.budgets.Load(taskID); ok { return v.(*agent.BudgetLedger) } - v, _ := e.budgets.LoadOrStore(taskID, agent.NewBudgetLedger(agent.BudgetsFromEnv())) - return v.(*agent.BudgetLedger) + created := agent.NewBudgetLedger(agent.BudgetsFromEnv()) + e.seedBudget(taskID, created) + v, loaded := e.budgets.LoadOrStore(taskID, created) + if loaded { + return v.(*agent.BudgetLedger) + } + return created +} + +// seedBudget rebuilds spent tokens and turns from llm_usage the first time a +// ledger is created in this process. The in-memory sink runs before the row is +// inserted, so a call that happens after this seed is counted once. A nil +// ledger means no budget is configured and nothing is read. +func (e *Engine) seedBudget(taskID string, ledger *agent.BudgetLedger) { + if ledger == nil || e == nil || e.m == nil || e.m.pg == nil || taskID == "" { + return + } + tokens, turns, err := e.m.pg.TaskLLMSpend(taskID) + if err != nil { + log.Printf("[budget] task %s: cannot restore spend from llm_usage: %v", taskID, err) + return + } + if tokens == 0 && turns == 0 { + return + } + ledger.RestoreSpend(agent.Spend{Tokens: tokens, Turns: turns}) + log.Printf("[budget] task %s restored %d tokens / %d turns from llm_usage", taskID, tokens, turns) } // budgetStop is the enforcement point of a planner round: it logs the remaining From 40de6ba8983b076101e80a97c1a3f3169b4b6610 Mon Sep 17 00:00:00 2001 From: protonmatter Date: Tue, 17 Nov 2026 15:50:00 -0500 Subject: [PATCH 2/3] Say what a restored task budget still allows, and test the lanes that count. A task already inside the wrap-up reserve only has that reserve left. The worker estimate starts from the defaults because usage rows have no run boundary. --- CHANGELOG.md | 1 + README.md | 13 ++++++-- agent/budget.go | 11 ++++--- db/task_llm_spend_test.go | 40 ++++++++++++++++++++++ server/engine_budget.go | 11 ++++--- server/engine_budget_test.go | 64 ++++++++++++++++++++++++++++++++++++ 6 files changed, 130 insertions(+), 10 deletions(-) create mode 100644 db/task_llm_spend_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5279232c8..554e42804 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ All notable changes to this project are documented in this file. The format is b ### Changed +- **Task budgets survive a restart**: planner, worker, judge, and compactor rows in `llm_usage` are counted back into `ARTEX_TASK_BUDGET_*`. A restored task already inside the wrap-up reserve only has that reserve left, so the run wraps up instead of starting another worker. The per-worker estimate is not restored and starts from the defaults (50,000 tokens and 10 turns) until a worker finishes in this process. Chat, goal decomposition, and side questions stay outside the cap. - **Untrusted-output boundary is now on by default**: Norma frames tool results for the worker, planner, chat and main agent unless `ARTEX_UNTRUSTED_BOUNDARY` is set to `0`, `false`, `off` or `no`. It was opt-in via `ARTEX_UNTRUSTED_BOUNDARY=1` in 0.6.0 (#166); the default flips here (#203). - **Versioned migrations**: the `db` package applies numbered `db/migrations/NNNN_slug.sql` files once, in order, under the schema advisory lock, and records a SHA-256 checksum in `schema_migrations`; missing, renamed, changed or out-of-order migrations fail closed (#226). - **Coverage graph performance**: the coverage graph batches its connector lookups (three queries instead of one per root and per company) and adds the partial index `idx_ts_icp`; on a task of about 3,000 assets the p50 dropped from 126 ms to 27 ms and the queries per call from 221 to 3 (#230). diff --git a/README.md b/README.md index ab353d543..ad8efec17 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,10 @@ worker remains confined to the intent it claimed. - **Budgets**: `ARTEX_TASK_BUDGET_TOKENS` / `ARTEX_TASK_BUDGET_TURNS` cap a whole task (planner and workers); `ARTEX_INTENT_BUDGET_TOKENS` / `ARTEX_INTENT_BUDGET_TURNS` cap one worker run. Near the limit the task enters a forced wrap-up; a worker is not started when the remaining budget cannot - cover it. Budgets are in memory: a restart starts fresh. + cover it. Spend already in `llm_usage` is restored after a restart. A task that is already inside + the wrap-up reserve only has that reserve left, so the revived run wraps up instead of starting + another worker. The per-worker cost estimate starts again from the defaults (50,000 tokens and + 10 turns) until a worker finishes in this process. - **Noise governor**: `ARTEX_NOISE_PROFILE=quiet|balanced|aggressive` paces requests ARTEX sends to targets (per-host rate, burst, in-flight limit, jitter, backoff on 429/503, short GET caching). With a profile set, agents also receive low-and-slow flags for the installed scanners. @@ -455,7 +458,13 @@ workers, else 50,000 tokens and 10 turns). Near the wrap-up reserve (the larger tokens, and 2 turns) the task enters a forced wrap-up: the planner writes a summary and final goal judgment, and workers may only write back. The task then ends `done` if its goals are met, else `timeout`. Practical floor: below 50,000 tokens no worker starts; 50,000-70,000 typically gives one -worker then wrap-up; pick well above 70,000 for real exploration. +worker then wrap-up; pick well above 70,000 for real exploration. Planner, worker, judge, and +compactor rows in `llm_usage` are counted back into the cap when a ledger is created, including +after a restart and after a revived run drops its ledger. A restored task already inside the +wrap-up reserve does not get another worker: only that reserve remains, and the run wraps up. +Chat, goal decomposition, and side questions are stored on the same task and are not counted. +The per-worker average is not restored (usage rows have no run boundary), so the estimate starts +from the defaults until a worker finishes in this process. **Common flags**: `./start.sh -addr :8787 -proxy :8788` (`-addr` frontend/API, `-proxy` traffic recording proxy). diff --git a/agent/budget.go b/agent/budget.go index 8fe727e55..4eb2925c0 100644 --- a/agent/budget.go +++ b/agent/budget.go @@ -92,7 +92,8 @@ func BudgetsFromEnv() (task, intent Budget) { NewBudget(env("ARTEX_INTENT_BUDGET_TOKENS"), env("ARTEX_INTENT_BUDGET_TURNS")) } -// BudgetLedger meters one task, in memory (a process restart starts afresh). +// BudgetLedger meters one task. The total is restored from llm_usage when the +// ledger is created, including after a restart. Per-worker averages are not. type BudgetLedger struct { task, intent Budget mu sync.Mutex @@ -110,9 +111,11 @@ func NewBudgetLedger(task, intent Budget) *BudgetLedger { return &BudgetLedger{task: task, intent: intent} } -// RestoreSpend adds usage that was already recorded before this process started. -// It is applied once, to a ledger that has not recorded anything yet. A nil -// ledger (no budget configured) is a no-op. +// RestoreSpend adds usage that was already recorded before this ledger existed. +// It only adds to the task total. Per-worker averages stay at the defaults +// until a worker finishes against this ledger, because llm_usage rows do not +// mark where one worker run ends and the next begins. A nil ledger (no budget +// configured) is a no-op. func (l *BudgetLedger) RestoreSpend(spent Spend) { if l == nil || (spent.Tokens == 0 && spent.Turns == 0) { return diff --git a/db/task_llm_spend_test.go b/db/task_llm_spend_test.go new file mode 100644 index 000000000..7c807f24b --- /dev/null +++ b/db/task_llm_spend_test.go @@ -0,0 +1,40 @@ +package db + +import ( + "fmt" + "testing" + "time" +) + +// Judge calls share the planner or worker sink and count toward the task cap. +// Chat rows for the same task do not, or a restart would tighten the cap. +func TestTaskLLMSpendCountsJudgeNotChat(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + t.Skipf("postgres unavailable (%v)", err) + } + defer d.Close() + + taskID := fmt.Sprintf("spend-%d", time.Now().UnixNano()) + t.Cleanup(func() { + _, _ = d.Exec(`DELETE FROM llm_usage WHERE task_id=$1`, taskID) + }) + insert := func(worker string, in, out int) { + t.Helper() + if err := d.InsertLLMUsage(&LLMUsage{ + TaskID: taskID, Worker: worker, InputTokens: in, OutputTokens: out, Status: "ok", + }); err != nil { + t.Fatal(err) + } + } + insert("judge", 30, 10) + insert("chat", 1000, 1000) + + tokens, turns, err := d.TaskLLMSpend(taskID) + if err != nil { + t.Fatal(err) + } + if tokens != 40 || turns != 1 { + t.Fatalf("tokens=%d turns=%d, want the judge row only (40 tokens, 1 turn)", tokens, turns) + } +} diff --git a/server/engine_budget.go b/server/engine_budget.go index a11ab7a59..fc6d45af3 100644 --- a/server/engine_budget.go +++ b/server/engine_budget.go @@ -26,10 +26,13 @@ func (e *Engine) budgetFor(taskID string) *agent.BudgetLedger { return created } -// seedBudget rebuilds spent tokens and turns from llm_usage the first time a -// ledger is created in this process. The in-memory sink runs before the row is -// inserted, so a call that happens after this seed is counted once. A nil -// ledger means no budget is configured and nothing is read. +// seedBudget rebuilds spent tokens and turns from llm_usage for every new +// ledger. budgetFor calls it on first use, and again after resetSettledRun or +// StopTask deletes the ledger in this same process. The in-memory sink runs +// before the row is inserted, so a call that happens after this seed is +// counted once. A nil ledger means no budget is configured and nothing is read. +// The restore does not rebuild per-worker averages, so the next worker estimate +// starts from the defaults until a worker finishes against this ledger. func (e *Engine) seedBudget(taskID string, ledger *agent.BudgetLedger) { if ledger == nil || e == nil || e.m == nil || e.m.pg == nil || taskID == "" { return diff --git a/server/engine_budget_test.go b/server/engine_budget_test.go index 26365bdcd..c3a8ebd43 100644 --- a/server/engine_budget_test.go +++ b/server/engine_budget_test.go @@ -2,9 +2,12 @@ package server import ( "context" + "strconv" "testing" + "time" "github.com/Autumn-27/artex/agent" + pgdb "github.com/Autumn-27/artex/db" "github.com/Autumn-27/norma/llm" ) @@ -67,3 +70,64 @@ func TestResetSettledRunRestartsBudget(t *testing.T) { t.Fatalf("revived run: ledger not fresh, spent %d tokens", got) } } + +// With Postgres, resetSettledRun drops the ledger and the next one is seeded +// again from llm_usage. In-memory spend that was never written does not come +// back. A restore that lands on the wrap-up reserve leaves only that reserve, +// and the worker estimate starts from the defaults. +func TestResetSettledRunReseedsBudgetFromUsage(t *testing.T) { + for _, k := range []string{"ARTEX_TASK_BUDGET_TURNS", "ARTEX_INTENT_BUDGET_TOKENS", "ARTEX_INTENT_BUDGET_TURNS"} { + t.Setenv(k, "") + } + t.Setenv("ARTEX_TASK_BUDGET_TOKENS", "100000") + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Skipf("postgres unavailable (%v)", err) + } + defer m.Close() + + taskID := strconv.FormatInt(time.Now().UnixNano(), 10) + t.Cleanup(func() { + _, _ = m.pg.Exec(`DELETE FROM llm_usage WHERE task_id=$1`, taskID) + }) + // 100000 - wrap-up reserve (max(5%, 20000)) = 80000. One planner row. + if err := m.pg.InsertLLMUsage(&pgdb.LLMUsage{ + TaskID: taskID, Worker: "planner", InputTokens: 80_000, Status: "ok", + }); err != nil { + t.Fatal(err) + } + e := NewEngine(m) + task := &Task{ID: taskID} + l := e.budgetFor(task.ID) + st := l.State() + if st.Spent.Tokens != 80_000 || st.Spent.Turns != 1 { + t.Fatalf("seed spent=%+v, want 80000 tokens and 1 turn", st.Spent) + } + if st.Remaining.Tokens != st.Margin.Tokens || !st.AtMargin || st.Dispatch { + t.Fatalf("restored task near the cap: remaining=%d margin=%d at=%v dispatch=%v", + st.Remaining.Tokens, st.Margin.Tokens, st.AtMargin, st.Dispatch) + } + if st.Estimate.Tokens != agent.DefaultWorkerEstimateTokens || st.Estimate.Turns != agent.DefaultWorkerEstimateTurns { + t.Fatalf("estimate=%+v, want the defaults (worker history is not restored)", st.Estimate) + } + + l.Record(llm.Usage{InputTokens: 1000}) + l.BeginWrapup() + e.settling.Store(task.ID, true) + e.resetSettledRun(task.ID) + if e.isSettling(task.ID) { + t.Fatal("revived run is still marked settling") + } + again := e.budgetFor(task.ID) + if again == l { + t.Fatal("revive reused the deleted ledger") + } + got := again.State() + if got.Spent.Tokens != 80_000 || got.Spent.Turns != 1 { + t.Fatalf("reseed spent=%+v, want the usage row and not the in-memory extra", got.Spent) + } + if got.Remaining.Tokens != got.Margin.Tokens || got.Dispatch { + t.Fatalf("reseed still past the wrap-up margin: remaining=%d margin=%d dispatch=%v", + got.Remaining.Tokens, got.Margin.Tokens, got.Dispatch) + } +} From 074962a351c5e56ad07ce6900b76fce7c48d6dc0 Mon Sep 17 00:00:00 2001 From: protonmatter Date: Tue, 17 Nov 2026 16:25:16 -0500 Subject: [PATCH 3/3] Say that a revived task keeps the spend already recorded. --- server/engine_timeout.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/engine_timeout.go b/server/engine_timeout.go index 37f5469f0..e2ae38425 100644 --- a/server/engine_timeout.go +++ b/server/engine_timeout.go @@ -93,7 +93,8 @@ func (e *Engine) resetTimeoutRevival(taskID string) { } // resetSettledRun clears the settle state of a finished run (settling and the -// budget ledger) so a revived task can plan and claim again with a fresh budget. +// budget ledger) so a revived task can plan and claim again. The next ledger +// is seeded from llm_usage, so the spend already recorded still counts. // It also clears the persisted end_reason of the previous run (ApplyTaskAdmission // already did in the same UPDATE as the revive; this is idempotent and never // touches a task that is terminal again).