Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions cmd/artex/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,24 @@ func run() int {
<-ctx.Done()

log.Println("shutting down...")
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = httpSrv.Shutdown(shutdownCtx)
// Close the listener before draining. Otherwise a request accepted during
// the drain can start a detached run, and mgr.Close runs while that run
// is still writing.
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
httpErr := httpSrv.Shutdown(shutdownCtx)
cancel()
if httpErr != nil {
log.Printf("http shutdown: %v", httpErr)
}
drainCtx, drainCancel := context.WithTimeout(context.Background(), 15*time.Second)
drainErr := srv.Wait(drainCtx)
drainCancel()
if drainErr != nil {
log.Printf("task drain: %v", drainErr)
}
if httpErr != nil || drainErr != nil {
return 1
}
return 0
}

Expand Down
118 changes: 118 additions & 0 deletions db/activity_outbox.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
package db

import (
"encoding/json"
"fmt"
"log"
)

// activityOutboxMaxAttempts is how many times a row may fail AppendActivity
// before it is dropped. One stuck row must not sit at the head of the queue
// and keep every later row from being replayed.
const activityOutboxMaxAttempts = 5

// activityOutboxPayload is Activity plus Detail. Activity.Detail is omitted
// from ordinary JSON (the list payload stays small); the outbox has to keep it
// so a replayed row is the same step that failed to store.
type activityOutboxPayload struct {
Activity
Detail string `json:"detail,omitempty"`
}

func marshalActivity(a Activity) ([]byte, error) {
return json.Marshal(activityOutboxPayload{Activity: a, Detail: a.Detail})
}

func unmarshalActivity(raw []byte) (Activity, error) {
var payload activityOutboxPayload
if err := json.Unmarshal(raw, &payload); err != nil {
return Activity{}, err
}
a := payload.Activity
a.Detail = payload.Detail
return a, nil
}

// EnqueueActivity keeps one activity that AppendActivity could not store.
func (s *ExplorationStore) EnqueueActivity(a Activity, cause string) error {
if s == nil || s.db == nil {
return fmt.Errorf("activity outbox: no store")
}
raw, err := marshalActivity(a)
if err != nil {
return err
}
_, err = s.db.Exec(`INSERT INTO activity_outbox (exploration_id, payload, last_error) VALUES ($1, $2::jsonb, $3)`, s.expID, string(raw), cause)
return err
}

// FlushActivityOutbox inserts queued activities in order and deletes each row
// that lands. The insert and the delete commit together, so a crash cannot
// leave the activity stored and the outbox row still queued. A row that still
// cannot be inserted stays, with its attempt count increased, and the flush
// continues with the next row. After activityOutboxMaxAttempts the row is
// dropped and logged. A payload that cannot be decoded is discarded.
func (s *ExplorationStore) FlushActivityOutbox() (int, error) {
if s == nil || s.db == nil {
return 0, fmt.Errorf("activity outbox: no store")
}
rows, err := s.db.Query(`SELECT id, payload, attempts FROM activity_outbox WHERE exploration_id=$1 ORDER BY id`, s.expID)
if err != nil {
return 0, err
}
defer rows.Close()
type item struct {
id int64
payload []byte
attempts int
}
var items []item
for rows.Next() {
var it item
if err := rows.Scan(&it.id, &it.payload, &it.attempts); err != nil {
return 0, err
}
items = append(items, it)
}
if err := rows.Err(); err != nil {
return 0, err
}
stored := 0
for _, it := range items {
a, err := unmarshalActivity(it.payload)
if err != nil {
if _, delErr := s.db.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); delErr != nil {
return stored, delErr
}
continue
}
tx, err := s.db.Begin()
if err != nil {
return stored, err
}
if _, err := s.appendActivity(tx, a); err != nil {
_ = tx.Rollback()
next := it.attempts + 1
if next >= activityOutboxMaxAttempts {
log.Printf("[activity] outbox %d for exploration %d dropped after %d attempts: %v", it.id, s.expID, next, err)
if _, delErr := s.db.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); delErr != nil {
return stored, delErr
}
continue
}
if _, updErr := s.db.Exec(`UPDATE activity_outbox SET attempts=$2, last_error=$3 WHERE id=$1`, it.id, next, err.Error()); updErr != nil {
return stored, updErr
}
continue
}
if _, err := tx.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); err != nil {
_ = tx.Rollback()
return stored, err
}
if err := tx.Commit(); err != nil {
return stored, err
}
stored++
}
return stored, nil
}
99 changes: 99 additions & 0 deletions db/activity_outbox_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package db

import "testing"

func TestActivityOutboxKeepsDetail(t *testing.T) {
in := Activity{Worker: "worker", Kind: "tool_result", Summary: "done", Detail: "full body", IsError: true}
raw, err := marshalActivity(in)
if err != nil {
t.Fatal(err)
}
out, err := unmarshalActivity(raw)
if err != nil {
t.Fatal(err)
}
if out.Detail != in.Detail || out.Summary != in.Summary || out.Worker != in.Worker || out.Kind != in.Kind || !out.IsError {
t.Fatalf("round trip = %+v", out)
}
}

func TestFlushActivityOutboxReplaysOnce(t *testing.T) {
d, err := Open(testDSN(t))
if err != nil {
t.Skipf("postgres unavailable (%v) — skipping", err)
}
defer d.Close()
expID, err := d.CreateExploration("outbox", "replay")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_, _ = d.Exec(`DELETE FROM activity_outbox WHERE exploration_id=$1`, expID)
_, _ = d.Exec(`DELETE FROM explorations WHERE id=$1`, expID)
})
es := d.Exploration(expID)
if err := es.EnqueueActivity(Activity{Worker: "worker", Kind: "text", Summary: "kept", Detail: "body"}, "insert failed"); err != nil {
t.Fatal(err)
}
n, err := es.FlushActivityOutbox()
if err != nil || n != 1 {
t.Fatalf("flush n=%d err=%v", n, err)
}
var activities, queued int
if err := d.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='kept'`, expID).Scan(&activities); err != nil {
t.Fatal(err)
}
if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, expID).Scan(&queued); err != nil {
t.Fatal(err)
}
if activities != 1 || queued != 0 {
t.Fatalf("activities=%d outbox=%d, want 1 and 0", activities, queued)
}
if n, err = es.FlushActivityOutbox(); err != nil || n != 0 {
t.Fatalf("second flush n=%d err=%v", n, err)
}
if err := d.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='kept'`, expID).Scan(&activities); err != nil {
t.Fatal(err)
}
if activities != 1 {
t.Fatalf("activities=%d after a second flush, want 1", activities)
}
}

func TestFlushActivityOutboxDropsStuckRow(t *testing.T) {
d, err := Open(testDSN(t))
if err != nil {
t.Skipf("postgres unavailable (%v) — skipping", err)
}
defer d.Close()
const missing = int64(9_000_000_000_001)
t.Cleanup(func() { _, _ = d.Exec(`DELETE FROM activity_outbox WHERE exploration_id=$1`, missing) })
es := d.Exploration(missing)
if err := es.EnqueueActivity(Activity{Worker: "worker", Kind: "text", Summary: "stuck"}, "no parent"); err != nil {
t.Fatal(err)
}
for attempt := 1; attempt < activityOutboxMaxAttempts; attempt++ {
n, err := es.FlushActivityOutbox()
if err != nil || n != 0 {
t.Fatalf("attempt %d: n=%d err=%v", attempt, n, err)
}
var queued int
if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, missing).Scan(&queued); err != nil {
t.Fatal(err)
}
if queued != 1 {
t.Fatalf("attempt %d: outbox rows=%d, want the stuck row kept", attempt, queued)
}
}
n, err := es.FlushActivityOutbox()
if err != nil || n != 0 {
t.Fatalf("final flush n=%d err=%v", n, err)
}
var queued int
if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, missing).Scan(&queued); err != nil {
t.Fatal(err)
}
if queued != 0 {
t.Fatalf("outbox rows=%d, want the stuck row dropped", queued)
}
}
17 changes: 15 additions & 2 deletions db/exploration.go
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,10 @@ WHERE exploration_id=$1 AND kind='intent' AND state='running'
if err != nil {
return 0, err
}
n, _ := res.RowsAffected()
n, err := res.RowsAffected()
if err != nil {
return 0, err
}
return n, nil
}

Expand Down Expand Up @@ -1379,14 +1382,24 @@ func (s *ExplorationStore) Stats() (map[string]int, error) {

// --- activity (poll by global id cursor) ---

// queryRower is the subset of *sql.DB and *sql.Tx that AppendActivity needs,
// so a replay can insert the activity and delete the outbox row together.
type queryRower interface {
QueryRow(query string, args ...any) *sql.Row
}

// AppendActivity records one worker step and returns its id.
func (s *ExplorationStore) AppendActivity(a Activity) (int64, error) {
return s.appendActivity(s.db, a)
}

func (s *ExplorationStore) appendActivity(q queryRower, a Activity) (int64, error) {
metadata := a.Metadata
if len(metadata) == 0 {
metadata = json.RawMessage(`{}`)
}
var id int64
err := s.db.QueryRow(`
err := q.QueryRow(`
INSERT INTO activity(exploration_id, node_id, worker, kind, tool, tool_use_id, is_error, summary, detail, metadata, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, main_seg)
VALUES ($1,$2,NULLIF($3,''),NULLIF($4,''),NULLIF($5,''),NULLIF($6,''),$7,NULLIF($8,''),NULLIF($9,''),$10,$11,$12,$13,$14,$15)
RETURNING id`, s.expID, a.NodeID, utf8Clean(a.Worker), utf8Clean(a.Kind), utf8Clean(a.Tool), utf8Clean(a.ToolUseID), a.IsError, utf8Clean(a.Summary), utf8Clean(a.Detail),
Expand Down
4 changes: 2 additions & 2 deletions db/migrate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,9 +137,9 @@ func assertEmbeddedApplied(t *testing.T, d *DB) []appliedMigration {
version int
name string
}{{1, llmRecordsMigration}, {2, llmUsageMigration}, {3, "0003_task_scope_icp_index.sql"}, {4, "0004_llm_usage_ts_index.sql"},
{5, "0005_traffic_rewrite_rules.sql"}}
{5, "0005_traffic_rewrite_rules.sql"}, {6, "0006_activity_outbox.sql"}}
if len(applied) != len(want) {
t.Fatalf("schema_migrations = %+v, want versions 1 to 5", applied)
t.Fatalf("schema_migrations = %+v, want versions 1 to 6", applied)
}
for i, w := range want {
content, err := fs.ReadFile(migrationsFS(), w.name)
Expand Down
15 changes: 15 additions & 0 deletions db/migrations/0006_activity_outbox.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
-- Activities that could not be inserted (the exploration row was missing, or
-- the write failed after retries) are kept here and replayed into activity
-- once the parent row accepts them. No foreign key: the row has to survive
-- the missing exploration that caused the original failure.
CREATE TABLE IF NOT EXISTS activity_outbox (
id BIGSERIAL PRIMARY KEY,
exploration_id BIGINT NOT NULL,
payload JSONB NOT NULL,
last_error TEXT,
attempts INT NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);

CREATE INDEX IF NOT EXISTS activity_outbox_exploration_idx
ON activity_outbox (exploration_id, id);
Loading
Loading