diff --git a/agents/agentd/go.mod b/agents/agentd/go.mod index db85372c..a63160a4 100644 --- a/agents/agentd/go.mod +++ b/agents/agentd/go.mod @@ -1,13 +1,24 @@ module github.com/agent-command/agentd -go 1.22 +go 1.22.0 require ( github.com/creack/pty v1.1.24 github.com/fsnotify/fsnotify v1.7.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 + github.com/prometheus/client_golang v1.22.0 gopkg.in/yaml.v3 v3.0.1 ) -require golang.org/x/sys v0.4.0 // indirect +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + golang.org/x/sys v0.30.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect +) diff --git a/agents/agentd/go.sum b/agents/agentd/go.sum index 457677c9..059fbf25 100644 --- a/agents/agentd/go.sum +++ b/agents/agentd/go.sum @@ -1,14 +1,50 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/agents/agentd/internal/metrics/metrics.go b/agents/agentd/internal/metrics/metrics.go new file mode 100644 index 00000000..c2a27283 --- /dev/null +++ b/agents/agentd/internal/metrics/metrics.go @@ -0,0 +1,98 @@ +package metrics + +import ( + "net/http" + "sync" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var once sync.Once + +var ( + wsConnected = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "agentd_ws_connected", + Help: "Whether agentd is currently connected to the control plane (1=yes, 0=no).", + }) + + wsReconnecting = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "agentd_ws_reconnecting", + Help: "Whether agentd is currently in a reconnect loop (1=yes, 0=no).", + }) + + wsReconnectAttemptsTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "agentd_ws_reconnect_attempts_total", + Help: "Total WebSocket reconnect attempts to the control plane.", + }) + + wsReconnectFailuresTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "agentd_ws_reconnect_failures_total", + Help: "Total WebSocket reconnect failures to the control plane.", + }) + + wsReconnectSuccessTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "agentd_ws_reconnect_success_total", + Help: "Total WebSocket reconnect successes to the control plane.", + }) + + wsReconnectBackoffSeconds = prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "agentd_ws_reconnect_backoff_seconds", + Help: "Backoff delay (sleep) used before each reconnect attempt.", + Buckets: []float64{0.25, 0.5, 1, 2, 5, 10, 30, 60}, + }) +) + +func initOnce() { + once.Do(func() { + prometheus.MustRegister( + wsConnected, + wsReconnecting, + wsReconnectAttemptsTotal, + wsReconnectFailuresTotal, + wsReconnectSuccessTotal, + wsReconnectBackoffSeconds, + ) + }) +} + +func Handler() http.Handler { + initOnce() + return promhttp.Handler() +} + +func SetWSConnected(connected bool) { + initOnce() + if connected { + wsConnected.Set(1) + return + } + wsConnected.Set(0) +} + +func SetWSReconnecting(reconnecting bool) { + initOnce() + if reconnecting { + wsReconnecting.Set(1) + return + } + wsReconnecting.Set(0) +} + +func RecordWSReconnectAttempt(delayMs int) { + initOnce() + wsReconnectAttemptsTotal.Inc() + if delayMs > 0 { + wsReconnectBackoffSeconds.Observe(float64(delayMs) / 1000.0) + } +} + +func RecordWSReconnectFailure() { + initOnce() + wsReconnectFailuresTotal.Inc() +} + +func RecordWSReconnectSuccess() { + initOnce() + wsReconnectSuccessTotal.Inc() +} diff --git a/agents/agentd/internal/providers/claude.go b/agents/agentd/internal/providers/claude.go index 211ed4de..5991c2e7 100644 --- a/agents/agentd/internal/providers/claude.go +++ b/agents/agentd/internal/providers/claude.go @@ -11,6 +11,7 @@ import ( "time" "github.com/agent-command/agentd/internal/config" + "github.com/agent-command/agentd/internal/metrics" "github.com/google/uuid" ) @@ -63,6 +64,7 @@ func (p *ClaudeProvider) Start() error { mux := http.NewServeMux() mux.HandleFunc("/v1/hooks/claude", p.handleHook) mux.HandleFunc("/v1/hooks/codex", p.handleCodexHook) + mux.Handle("/metrics", metrics.Handler()) p.server = &http.Server{ Addr: p.cfg.HooksHTTPListen, diff --git a/agents/agentd/internal/ws/client.go b/agents/agentd/internal/ws/client.go index ec3208cc..74111def 100644 --- a/agents/agentd/internal/ws/client.go +++ b/agents/agentd/internal/ws/client.go @@ -10,27 +10,28 @@ import ( "sync/atomic" "time" - "github.com/gorilla/websocket" + "github.com/agent-command/agentd/internal/metrics" "github.com/agent-command/agentd/internal/queue" + "github.com/gorilla/websocket" ) type MessageHandler func(msgType string, payload json.RawMessage) type Client struct { - url string - token string - hostID string - backoff []int - conn *websocket.Conn - mu sync.Mutex - seq atomic.Int64 - lastAckedSeq int64 - onMessage MessageHandler - done chan struct{} - reconnecting bool - queue *queue.Queue - stateDir string - onConnect func() + url string + token string + hostID string + backoff []int + conn *websocket.Conn + mu sync.Mutex + seq atomic.Int64 + lastAckedSeq int64 + onMessage MessageHandler + done chan struct{} + reconnecting bool + queue *queue.Queue + stateDir string + onConnect func() } func NewClient(url, token, hostID string, backoff []int) *Client { @@ -76,6 +77,8 @@ func (c *Client) Connect() error { c.conn = conn c.reconnecting = false + metrics.SetWSConnected(true) + metrics.SetWSReconnecting(false) // Start reader goroutine go c.reader() @@ -95,6 +98,8 @@ func (c *Client) reader() { } c.mu.Unlock() + metrics.SetWSConnected(false) + // Attempt reconnection c.reconnect() }() @@ -174,6 +179,9 @@ func (c *Client) reconnect() { c.reconnecting = true c.mu.Unlock() + metrics.SetWSReconnecting(true) + defer metrics.SetWSReconnecting(false) + for i, delay := range c.backoff { select { case <-c.done: @@ -181,12 +189,15 @@ func (c *Client) reconnect() { case <-time.After(time.Duration(delay) * time.Millisecond): } + metrics.RecordWSReconnectAttempt(delay) log.Printf("Reconnection attempt %d/%d", i+1, len(c.backoff)) if err := c.Connect(); err == nil { + metrics.RecordWSReconnectSuccess() log.Printf("Reconnected successfully") return } + metrics.RecordWSReconnectFailure() } // Keep trying with max backoff @@ -198,10 +209,13 @@ func (c *Client) reconnect() { case <-time.After(time.Duration(maxDelay) * time.Millisecond): } + metrics.RecordWSReconnectAttempt(maxDelay) if err := c.Connect(); err == nil { + metrics.RecordWSReconnectSuccess() log.Printf("Reconnected successfully") return } + metrics.RecordWSReconnectFailure() } } diff --git a/docs/README.md b/docs/README.md index 7c66318c..70b09045 100644 --- a/docs/README.md +++ b/docs/README.md @@ -35,5 +35,6 @@ Reference: Operations: - [Operations](operations.md) +- [Reliability Primitives](reliability-primitives.md) - [Troubleshooting](troubleshooting.md) - [Implementation Notes](implementation.md) diff --git a/docs/agentd.md b/docs/agentd.md index de10ffe8..03c3bccc 100644 --- a/docs/agentd.md +++ b/docs/agentd.md @@ -54,6 +54,13 @@ Disable these if you want read only mode on a host. agentd can handle provider specific hooks and usage parsing. See [Provider Hooks](hooks.md) for setup details. +### Metrics + +agentd exposes Prometheus metrics at `GET /metrics` on the same HTTP listener as provider hooks (`providers.claude.hooks_http_listen`). + +- Default: `http://127.0.0.1:7777/metrics` +- If you bind hooks to a non-loopback address, treat `/metrics` as sensitive (firewall or reverse proxy auth). + ### Provider usage polling agentd can run periodic commands to pull usage data from provider CLIs or APIs diff --git a/docs/operations.md b/docs/operations.md index e5427828..c3fddf02 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -10,6 +10,7 @@ This guide covers operational best practices for running Agent Commander. ## Monitoring - Watch `/health` for control plane status. +- Scrape `/metrics` for operational counters (reconnect/backoff, Clawdbot rate-limit decisions). If `METRICS_TOKEN` is set on the control plane, Prometheus must send `Authorization: Bearer ` (or `x-metrics-token`). - Monitor database connections and disk usage. - Track agentd logs for reconnect loops or tmux errors. diff --git a/docs/reliability-primitives.md b/docs/reliability-primitives.md new file mode 100644 index 00000000..c7558931 --- /dev/null +++ b/docs/reliability-primitives.md @@ -0,0 +1,60 @@ +# Reliability Primitives + +This doc inventories how **retry/backoff**, **rate limiting**, and **ops metrics** currently work in Agent Command, and proposes a migration path toward consistent, observable behavior. + +## Inventory (Current State) + +### Retry + Backoff + +**agentd (Go)** +- WebSocket reconnect loop: `agents/agentd/internal/ws/client.go` + - Policy: fixed backoff schedule from config `control_plane.reconnect_backoff_ms` (default `[250, 500, 1000, 2000, 5000]` ms) and then retries forever at the max delay. + - No jitter. + - No per-attempt timeout (relies on the underlying dial behavior). +- Ad-hoc “retry once” for Claude usage command parsing: `agents/agentd/cmd/agentd/main.go` (`maybeRetryClaudeUsage`) + - If output looks incomplete, re-runs the command using `script -c` to force a TTY. + - No backoff. + +**control-plane (TypeScript)** +- No shared retry helper (outbound HTTP calls generally rely on per-call timeouts). +- One “retry for race” pattern exists in group creation logic (check-then-recheck on unique collision): `services/control-plane/src/routes/groups.ts` and `services/control-plane/src/db/index.ts`. + +### Rate Limiting + +**control-plane (TypeScript)** +- In-memory per-user throttle for Clawdbot notifications: `services/control-plane/src/services/clawdbot.ts` + - Default: `maxPerHour = 30` with additional dedupe + session cooldown rules. + - Not distributed-safe (assumes a single control-plane instance for correct enforcement). + +### Metrics / Observability + +**Before 2026-02-16:** operational metrics (Prometheus-style) were not exposed; only session analytics existed in Postgres (`migrations/009_session_analytics.sql`). + +**After 2026-02-16:** operational `/metrics` endpoints exist: +- control-plane: `GET /metrics` (token optional via `METRICS_TOKEN`) +- agentd: `GET /metrics` served from the hooks HTTP server (same listen address as `providers.claude.hooks_http_listen`) + +## Standard Primitives (Recommended) + +### Go (agentd) +1. **Retry/backoff standard:** exponential backoff with jitter and a hard cap. +2. **Timeout standard:** every outbound operation should be bounded by context deadlines. +3. **Metrics standard:** Prometheus counters/histograms for reconnect attempts, failures, and backoff delays. + +### TypeScript (control-plane) +1. **Retry/backoff standard:** one shared helper (jitter, caps, per-attempt timeout, and explicit “non-retryable” classification). +2. **Rate limiting standard:** if control-plane is ever scaled horizontally, move throttles to Redis-backed primitives (in-memory is fine only when single-instance is guaranteed). +3. **Metrics standard:** Prometheus counters/histograms for decisioning (allowed/blocked + reason) and key outbound call failures. + +## Migration Plan (Incremental) + +1. **(Done) Add operational metrics endpoints and key counters** + - control-plane `/metrics` + Clawdbot allow/deny reasons + - agentd `/metrics` + reconnect/backoff counters +2. **Centralize retry/backoff** + - Introduce a small `retry` helper in each language (repo-local), then migrate high-value call sites first. +3. **Rate limiting hardening (only if/when needed)** + - If control-plane becomes multi-instance: add Redis and migrate Clawdbot throttle to distributed-safe limits. +4. **Dashboards/alerts** + - Alert on reconnect storms, reconnect failure rates, and sustained Clawdbot rate-limit blocks. + diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5e779066..c18a501c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -100,10 +100,10 @@ importers: version: 0.460.0(react@18.3.1) next: specifier: ^14.2.0 - version: 14.2.35(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + version: 14.2.35(@opentelemetry/api@1.9.0)(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) next-auth: specifier: ^4.24.0 - version: 4.24.13(next@14.2.35(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + version: 4.24.13(next@14.2.35(@opentelemetry/api@1.9.0)(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1) react: specifier: ^18.3.0 version: 18.3.1 @@ -214,6 +214,9 @@ importers: pino-pretty: specifier: ^10.3.1 version: 10.3.1 + prom-client: + specifier: ^15.1.3 + version: 15.1.3 ulid: specifier: ^2.3.0 version: 2.4.0 @@ -619,6 +622,10 @@ packages: resolution: {integrity: sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==} engines: {node: '>=12.4.0'} + '@opentelemetry/api@1.9.0': + resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} + engines: {node: '>=8.0.0'} + '@panva/hkdf@1.2.1': resolution: {integrity: sha512-6oclG6Y3PiDFcoyk8srjLfVKyMfVCKJ27JwNPViuXziFpmdz+MZnZN/aKY0JGXgYuO/VghU0jcOAZgWXZ1Dmrw==} @@ -1560,6 +1567,9 @@ packages: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} + bintrees@1.0.2: + resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} + brace-expansion@1.1.12: resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} @@ -3032,6 +3042,10 @@ packages: resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} engines: {node: '>= 0.6.0'} + prom-client@15.1.3: + resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==} + engines: {node: ^16 || ^18 || >=20} + promise-worker-transferable@1.0.4: resolution: {integrity: sha512-bN+0ehEnrXfxV2ZQvU2PetO0n4gqBD4ulq3MI1WOPLgr7/Mg9yRQkX5+0v1vagr74ZTsl7XtzlaYDo2EuCeYJw==} @@ -3434,6 +3448,9 @@ packages: engines: {node: '>=14.0.0'} hasBin: true + tdigest@0.1.2: + resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} + text-table@0.2.0: resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==} @@ -4120,6 +4137,8 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} + '@opentelemetry/api@1.9.0': {} + '@panva/hkdf@1.2.1': {} '@pinojs/redact@0.4.0': {} @@ -5099,6 +5118,8 @@ snapshots: binary-extensions@2.3.0: {} + bintrees@1.0.2: {} + brace-expansion@1.1.12: dependencies: balanced-match: 1.0.2 @@ -6614,13 +6635,13 @@ snapshots: natural-compare@1.4.0: {} - next-auth@4.24.13(next@14.2.35(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + next-auth@4.24.13(next@14.2.35(@opentelemetry/api@1.9.0)(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: '@babel/runtime': 7.28.6 '@panva/hkdf': 1.2.1 cookie: 0.7.2 jose: 4.15.9 - next: 14.2.35(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + next: 14.2.35(@opentelemetry/api@1.9.0)(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) oauth: 0.9.15 openid-client: 5.7.1 preact: 10.28.2 @@ -6629,7 +6650,7 @@ snapshots: react-dom: 18.3.1(react@18.3.1) uuid: 8.3.2 - next@14.2.35(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + next@14.2.35(@opentelemetry/api@1.9.0)(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: '@next/env': 14.2.35 '@swc/helpers': 0.5.5 @@ -6650,6 +6671,7 @@ snapshots: '@next/swc-win32-arm64-msvc': 14.2.33 '@next/swc-win32-ia32-msvc': 14.2.33 '@next/swc-win32-x64-msvc': 14.2.33 + '@opentelemetry/api': 1.9.0 '@playwright/test': 1.57.0 transitivePeerDependencies: - '@babel/core' @@ -6960,6 +6982,11 @@ snapshots: process@0.11.10: {} + prom-client@15.1.3: + dependencies: + '@opentelemetry/api': 1.9.0 + tdigest: 0.1.2 + promise-worker-transferable@1.0.4: dependencies: is-promise: 2.2.2 @@ -7456,6 +7483,10 @@ snapshots: - tsx - yaml + tdigest@0.1.2: + dependencies: + bintrees: 1.0.2 + text-table@0.2.0: {} thenify-all@1.6.0: diff --git a/project_status.md b/project_status.md index 38e4fda5..a9b0007b 100644 --- a/project_status.md +++ b/project_status.md @@ -1,7 +1,7 @@ # Project Status - agent-command > **Auto-generated by [project-status agent](https://github.com/cvsloane/open-agents)** -> Last updated: 2026-02-14T16:02:02.372Z +> Last updated: 2026-02-16T16:02:03.813Z ## Purpose @@ -18,27 +18,25 @@ This file provides context for AI assistants (Claude Code, Cursor, etc.) about r **Recent Terminal Output**: ``` - now only seeded by OrchestratorButton. I need to verify that OrchestratorModal is properly positioned and that its z-50 is above - the header, which doesn't have a z-index. I may need to update documentation to reflect that OrchestratorButton is now mounted. - Assessing improvements - The user wants improvements assessed, and I see this as a major one! I might think about updating the tasks note or creating a new - one for additional improvements, but it’s not strictly necessary. Next, I should rerun the typecheck since the lint and tests have - already passed before. It’ll be good to check the git diff to make sure everything looks right. I’ll quickly review the diff for - the orchestrator store and TerminalView to confirm there are no mistakes. Let's open that up! -• Ran cd /home/cvsloane/dev/agent-command && git diff -- apps/dashboard/src/stores/orchestrator.ts | sed -n '1,220p' - └ diff --git a/apps/dashboard/src/stores/orchestrator.ts b/apps/dashboard/src/stores/orchestrator.ts - index a434596..93fe002 100644 - … +93 lines - const snapshotContext = buildSnapshotContext(captureText); -• Ran cd /home/cvsloane/dev/agent-command && git diff -- apps/dashboard/src/components/TerminalView.tsx | sed -n '1,260p' - └ diff --git a/apps/dashboard/src/components/TerminalView.tsx b/apps/dashboard/src/components/TerminalView.tsx - index 1e3dd31..4336efa 100644 - … +188 lines - )} - aria-label="Interactive terminal" -◦ Assessing improvements (16m 16s • esc to interrupt) + testing scrolling on mobile. We can also verify if the orchestrator opens correctly. I'll keep this part minimal to avoid + overwhelming the user. Now, I’ll craft the final response to include these manual checks along with the automated testing + discussion we've had earlier. +─ Worked for 1m 56s ───────────────────────────────────────────────────────────────────────────────────────────────────────────────── +• Yes, but they’re mostly “build-time” guardrails, not functional UI regression tests. + What we have today (automated) + - CI runs pnpm lint, pnpm typecheck, pnpm build (so TS/ESLint/Next build failures get caught). .github/workflows/ci.yml + - CI runs Go unit tests for agents/agentd (go test ./...). .github/workflows/ci.yml + - CI also does Docker builds + CodeQL + dependency review. + What we don’t have + - No unit/integration test runner for the dashboard/control-plane packages (they don’t even define a test script), so pnpm test + doesn’t exercise UI logic. + - No Playwright/Cypress-style e2e tests in CI for orchestrator/terminal interactions. Playwright exists, but it’s currently just + perf scripts under scripts/perf/, not regression coverage for these changes. + If you want, I can add real regression coverage next (either Vitest unit tests for the orchestrator store + extracted terminal + gesture logic, or Playwright smoke tests + a CI job). +⚠ Heads up, you have less than 25% of your weekly limit left. Run /status for a breakdown. › Summarize recent commits - ? for shortcuts 45% context left + ? for shortcuts 60% context left ``` @@ -48,42 +46,42 @@ This file provides context for AI assistants (Claude Code, Cursor, etc.) about r | Hash | Message | Date | Author | |------|---------|------|--------| -| `9d4e0a0` | Merge pull request #44 from cvsloane/fix/codex-usage-buckets | 2026-02-13 | Chris Sloane | -| `691109b` | fix: codex-usage prefer non-spark limit bucket | 2026-02-13 | cvsloane | -| `91bc7f0` | fix: prefer non-spark codex usage + show spark breakdown | 2026-02-13 | cvsloane | -| `ffd866c` | Merge pull request #43 from cvsloane/fix/streaming-migrations | 2026-02-07 | Chris Sloane | -| `2b1a557` | fix: restore agent streaming by fixing migrations | 2026-02-07 | cvsloane | +| `677fadf` | fix(agentd): keep deps Go 1.22 compatible | 2026-02-16 | cvsloane | +| `eaefe89` | fix(agentd): pin prometheus client to Go 1.22 | 2026-02-16 | cvsloane | +| `956de1b` | feat(metrics): add /metrics and reliability inventory | 2026-02-16 | cvsloane | +| `38bddd7` | chore: update project status | 2026-02-15 | cvsloane | +| `877c499` | Merge pull request #49 from cvsloane/chore/tasks-note-followups | 2026-02-14 | Chris Sloane | --- ## Files Changed Recently -- `agents/agentd/cmd/agentd/main.go` -- `agents/agentd/cmd/agentd/main_usage_parse_test.go` - `agents/agentd/go.mod` - `agents/agentd/go.sum` -- `agents/agentd/internal/console/stream.go` -- `agents/agentd/internal/queue/queue.go` -- `apps/dashboard/src/app/(dashboard)/hosts/page.tsx` -- `apps/dashboard/src/app/(dashboard)/orchestrator/OrchestratorPageClient.tsx` -- `apps/dashboard/src/app/(dashboard)/page.tsx` +- `agents/agentd/internal/metrics/metrics.go` +- `agents/agentd/internal/providers/claude.go` +- `agents/agentd/internal/ws/client.go` - `apps/dashboard/src/app/(dashboard)/sessions/[id]/page.tsx` -- `apps/dashboard/src/app/(dashboard)/sessions/SessionsPageClient.tsx` -- `apps/dashboard/src/components/analytics/ProviderUtilization.tsx` -- `apps/dashboard/src/components/BulkActionToolbar.tsx` -- `apps/dashboard/src/components/SessionCard.tsx` -- `apps/dashboard/src/components/SessionList.tsx` -- `apps/dashboard/src/components/VoiceInputButton.tsx` -- `apps/dashboard/src/hooks/useOrchestratorSummaries.ts` -- `apps/dashboard/src/hooks/useTerminateSession.ts` -- `apps/dashboard/src/lib/api.ts` -- `apps/dashboard/src/lib/utils.ts` +- `apps/dashboard/src/components/orchestrator/OrchestratorButton.tsx` +- `apps/dashboard/src/components/TerminalView.tsx` +- `docs/agentd.md` +- `docs/operations.md` +- `docs/README.md` +- `docs/reliability-primitives.md` +- `pnpm-lock.yaml` +- `project_status.md` +- `services/control-plane/.env.example` +- `services/control-plane/package.json` +- `services/control-plane/src/config.ts` +- `services/control-plane/src/index.ts` +- `services/control-plane/src/metrics.ts` +- `services/control-plane/src/routes/metrics.ts` --- ## Branch Info -- **Generated From Branch**: `chore/project-status` -- **Last Commit**: 2026-02-13T14:16:53-05:00 +- **Current Branch**: `feat/ops-metrics` +- **Last Commit**: 2026-02-16T10:58:07-05:00 diff --git a/services/control-plane/.env.example b/services/control-plane/.env.example index 82ce1017..b8ccba0a 100644 --- a/services/control-plane/.env.example +++ b/services/control-plane/.env.example @@ -14,6 +14,12 @@ DATABASE_URL= HOST=0.0.0.0 PORT=8080 +# Optional: protect `GET /metrics` with a bearer token. +# If set, provide either: +# - `Authorization: Bearer ` +# - `x-metrics-token: ` +METRICS_TOKEN= + # ----------------------------------------------------------------------------- # SECURITY # ----------------------------------------------------------------------------- diff --git a/services/control-plane/package.json b/services/control-plane/package.json index 024461a9..25d50f67 100644 --- a/services/control-plane/package.json +++ b/services/control-plane/package.json @@ -25,6 +25,7 @@ "openai": "^6.16.0", "pg": "^8.13.0", "pino-pretty": "^10.3.1", + "prom-client": "^15.1.3", "ulid": "^2.3.0", "zod": "^3.23.0" }, diff --git a/services/control-plane/src/config.ts b/services/control-plane/src/config.ts index 71464ee9..939b81cf 100644 --- a/services/control-plane/src/config.ts +++ b/services/control-plane/src/config.ts @@ -6,6 +6,7 @@ const ConfigSchema = z.object({ HOST: z.string().default('0.0.0.0'), PORT: z.coerce.number().default(8080), JWT_SECRET: z.string().min(16), + METRICS_TOKEN: z.string().min(1).optional(), TAILNET_DOMAIN: z.string().optional(), OPENAI_API_KEY: z.string().optional(), OPENAI_MODEL: z.string().default('gpt-4o-mini'), diff --git a/services/control-plane/src/index.ts b/services/control-plane/src/index.ts index 136bdf82..a3fc9ef4 100644 --- a/services/control-plane/src/index.ts +++ b/services/control-plane/src/index.ts @@ -20,6 +20,7 @@ import { registerProjectRoutes } from './routes/projects.js'; import { registerSummaryRoutes } from './routes/summaries.js'; import { registerSettingsRoutes } from './routes/settings.js'; import { registerNotificationRoutes } from './routes/notifications.js'; +import { registerMetricsRoutes } from './routes/metrics.js'; import { pubsub } from './services/pubsub.js'; import { verifyRequestToken } from './auth/verify.js'; @@ -62,7 +63,7 @@ async function start(): Promise { // Auth for REST (skip health + WS routes that handle their own auth) app.addHook('onRequest', async (request, reply) => { const url = request.url; - if (url.startsWith('/health') || url.startsWith('/v1/agent/connect') || url.startsWith('/v1/ui/stream') || url.startsWith('/v1/ui/terminal') || url.startsWith('/v1/voice/transcribe')) { + if (url.startsWith('/health') || url.startsWith('/metrics') || url.startsWith('/v1/agent/connect') || url.startsWith('/v1/ui/stream') || url.startsWith('/v1/ui/terminal') || url.startsWith('/v1/voice/transcribe')) { return; } @@ -103,6 +104,7 @@ async function start(): Promise { registerSummaryRoutes(app); registerSettingsRoutes(app); registerNotificationRoutes(app); + registerMetricsRoutes(app); // Health check endpoint app.get('/health', async () => { diff --git a/services/control-plane/src/metrics.ts b/services/control-plane/src/metrics.ts new file mode 100644 index 00000000..0cc95c2f --- /dev/null +++ b/services/control-plane/src/metrics.ts @@ -0,0 +1,39 @@ +import client from 'prom-client'; + +export const registry = new client.Registry(); +registry.setDefaultLabels({ app: 'agent-command-control-plane' }); + +client.collectDefaultMetrics({ register: registry }); + +export type ClawdbotDecision = 'allowed' | 'blocked'; +export type ClawdbotReason = + | 'allowed' + | 'provider_disabled' + | 'actionable_only' + | 'event_disabled' + | 'rate_limit' + | 'approval_dedup' + | 'dedupe_key' + | 'session_cooldown'; + +export const clawdbotNotificationDecisionsTotal = new client.Counter({ + name: 'agent_command_clawdbot_notification_decisions_total', + help: 'Count of Clawdbot notification decisions (allowed/blocked) and reasons.', + labelNames: ['decision', 'reason', 'event_type', 'provider'] as const, + registers: [registry], +}); + +export function recordClawdbotNotificationDecision(params: { + decision: ClawdbotDecision; + reason: ClawdbotReason; + eventType: string; + provider: string | null; +}): void { + clawdbotNotificationDecisionsTotal.inc({ + decision: params.decision, + reason: params.reason, + event_type: params.eventType, + provider: params.provider ?? 'none', + }); +} + diff --git a/services/control-plane/src/routes/metrics.ts b/services/control-plane/src/routes/metrics.ts new file mode 100644 index 00000000..ecea7995 --- /dev/null +++ b/services/control-plane/src/routes/metrics.ts @@ -0,0 +1,35 @@ +import type { FastifyInstance } from 'fastify'; +import { config } from '../config.js'; +import { registry } from '../metrics.js'; + +function getBearerToken(authHeader: string | undefined): string | null { + if (!authHeader) return null; + if (!authHeader.startsWith('Bearer ')) return null; + return authHeader.slice('Bearer '.length); +} + +function getHeaderToken(value: unknown): string | null { + if (typeof value === 'string') return value; + if (Array.isArray(value) && typeof value[0] === 'string') return value[0]; + return null; +} + +export function registerMetricsRoutes(app: FastifyInstance): void { + app.get('/metrics', async (request, reply) => { + const expected = config.METRICS_TOKEN; + if (expected) { + const bearer = getBearerToken(request.headers.authorization); + const direct = getHeaderToken(request.headers['x-metrics-token']); + const provided = bearer ?? direct; + if (provided !== expected) { + return reply.status(401).send('Unauthorized\n'); + } + } + + const body = await registry.metrics(); + reply.header('Content-Type', registry.contentType); + reply.header('Cache-Control', 'no-store'); + return body; + }); +} + diff --git a/services/control-plane/src/services/clawdbot.ts b/services/control-plane/src/services/clawdbot.ts index 609ed264..2e975aba 100644 --- a/services/control-plane/src/services/clawdbot.ts +++ b/services/control-plane/src/services/clawdbot.ts @@ -1,6 +1,7 @@ import type { Session, Approval } from '@agent-command/schema'; import * as db from '../db/index.js'; import { pool } from '../db/index.js'; +import { recordClawdbotNotificationDecision, type ClawdbotReason } from '../metrics.js'; interface ClawdbotThrottleSettings { maxPerHour: number; @@ -32,6 +33,12 @@ interface RateLimitEntry { windowStart: number; } +type NotifyBlockReason = Exclude; + +type NotifyDecision = + | { allowed: true } + | { allowed: false; reason: { code: NotifyBlockReason; detail: string } }; + // Default events: only critical ones enabled const DEFAULT_CLAWDBOT_EVENTS: Record = { approvals: true, @@ -127,7 +134,7 @@ class ClawdbotNotifier { approvalId?: string; status?: string; } = {} - ): { allowed: boolean; reason?: string } { + ): NotifyDecision { const { isActionable, sessionId, approvalId, status } = options; const throttle = config.throttle ?? DEFAULT_THROTTLE; const actionable = isActionable !== false; @@ -135,23 +142,35 @@ class ClawdbotNotifier { // 1. Check actionableOnly filter if (actionableOnly && !actionable) { - return { allowed: false, reason: 'actionableOnly filter (not actionable)' }; + return { + allowed: false, + reason: { code: 'actionable_only', detail: 'actionableOnly filter (not actionable)' }, + }; } // 2. Check event type filter const eventEnabled = config.events[eventType] ?? DEFAULT_CLAWDBOT_EVENTS[eventType] ?? true; if (!eventEnabled) { - return { allowed: false, reason: `event type '${eventType}' disabled` }; + return { + allowed: false, + reason: { code: 'event_disabled', detail: `event type '${eventType}' disabled` }, + }; } // 3. Check rate limit (non-mutating) if (!this.canSend(userId, throttle.maxPerHour)) { - return { allowed: false, reason: 'rate limit exceeded' }; + return { + allowed: false, + reason: { code: 'rate_limit', detail: 'rate limit exceeded' }, + }; } // 4. Approval-specific deduplication if (approvalId && this.approvalNotified.has(this.approvalKey(userId, approvalId))) { - return { allowed: false, reason: `approval ${approvalId} already notified` }; + return { + allowed: false, + reason: { code: 'approval_dedup', detail: `approval ${approvalId} already notified` }, + }; } // 5. Strong deduplication by dedupeKey @@ -160,7 +179,10 @@ class ClawdbotNotifier { if (lastNotified) { const dedupeWindowMs = 5 * 60 * 1000; // 5 minutes if (Date.now() - lastNotified < dedupeWindowMs) { - return { allowed: false, reason: `duplicate notification (key: ${dedupeKey})` }; + return { + allowed: false, + reason: { code: 'dedupe_key', detail: `duplicate notification (key: ${dedupeKey})` }, + }; } } @@ -168,7 +190,10 @@ class ClawdbotNotifier { if (sessionId && !actionable) { const sessionLastTime = this.sessionLastNotified.get(this.sessionKey(userId, sessionId)); if (sessionLastTime && Date.now() - sessionLastTime < throttle.sessionCooldownMs) { - return { allowed: false, reason: `session ${sessionId} cooldown active` }; + return { + allowed: false, + reason: { code: 'session_cooldown', detail: `session ${sessionId} cooldown active` }, + }; } } @@ -217,16 +242,37 @@ class ClawdbotNotifier { // Check provider filter if (provider) { const providerEnabled = config.providers[provider] ?? true; - if (!providerEnabled) return; + if (!providerEnabled) { + recordClawdbotNotificationDecision({ + decision: 'blocked', + reason: 'provider_disabled', + eventType, + provider, + }); + return; + } } // Run shouldNotify checks - const { allowed, reason } = this.shouldNotify(userId, eventType, config, options); - if (!allowed) { - console.log(`[clawdbot] Notification blocked for user ${userId}: ${reason}`); + const decision = this.shouldNotify(userId, eventType, config, options); + if (!decision.allowed) { + recordClawdbotNotificationDecision({ + decision: 'blocked', + reason: decision.reason.code, + eventType, + provider, + }); + console.log(`[clawdbot] Notification blocked for user ${userId}: ${decision.reason.detail}`); return; } + recordClawdbotNotificationDecision({ + decision: 'allowed', + reason: 'allowed', + eventType, + provider, + }); + const throttle = config.throttle ?? DEFAULT_THROTTLE; // Consume rate limit only after passing all checks