From 38bddd715a097d1c2044df99f85b4663d43a06d1 Mon Sep 17 00:00:00 2001 From: cvsloane Date: Sun, 15 Feb 2026 11:02:03 -0500 Subject: [PATCH 1/6] chore: update project status --- project_status.md | 77 ++++++++++++++++++----------------------------- 1 file changed, 30 insertions(+), 47 deletions(-) diff --git a/project_status.md b/project_status.md index 38e4fda5..0196bb9c 100644 --- a/project_status.md +++ b/project_status.md @@ -1,7 +1,7 @@ # Project Status - agent-command > **Auto-generated by [project-status agent](https://github.com/cvsloane/open-agents)** -> Last updated: 2026-02-14T16:02:02.372Z +> Last updated: 2026-02-15T16:02:03.149Z ## Purpose @@ -18,27 +18,25 @@ This file provides context for AI assistants (Claude Code, Cursor, etc.) about r **Recent Terminal Output**: ``` - now only seeded by OrchestratorButton. I need to verify that OrchestratorModal is properly positioned and that its z-50 is above - the header, which doesn't have a z-index. I may need to update documentation to reflect that OrchestratorButton is now mounted. - Assessing improvements - The user wants improvements assessed, and I see this as a major one! I might think about updating the tasks note or creating a new - one for additional improvements, but it’s not strictly necessary. Next, I should rerun the typecheck since the lint and tests have - already passed before. It’ll be good to check the git diff to make sure everything looks right. I’ll quickly review the diff for - the orchestrator store and TerminalView to confirm there are no mistakes. Let's open that up! -• Ran cd /home/cvsloane/dev/agent-command && git diff -- apps/dashboard/src/stores/orchestrator.ts | sed -n '1,220p' - └ diff --git a/apps/dashboard/src/stores/orchestrator.ts b/apps/dashboard/src/stores/orchestrator.ts - index a434596..93fe002 100644 - … +93 lines - const snapshotContext = buildSnapshotContext(captureText); -• Ran cd /home/cvsloane/dev/agent-command && git diff -- apps/dashboard/src/components/TerminalView.tsx | sed -n '1,260p' - └ diff --git a/apps/dashboard/src/components/TerminalView.tsx b/apps/dashboard/src/components/TerminalView.tsx - index 1e3dd31..4336efa 100644 - … +188 lines - )} - aria-label="Interactive terminal" -◦ Assessing improvements (16m 16s • esc to interrupt) + I should mention a few manual checks we can perform: opening the session page, toggling fullscreen, checking for scroll lock, and + testing scrolling on mobile. We can also verify if the orchestrator opens correctly. I'll keep this part minimal to avoid + overwhelming the user. Now, I’ll craft the final response to include these manual checks along with the automated testing + discussion we've had earlier. +─ Worked for 1m 56s ───────────────────────────────────────────────────────────────────────────────────────────────────────────────── +• Yes, but they’re mostly “build-time” guardrails, not functional UI regression tests. + What we have today (automated) + - CI runs pnpm lint, pnpm typecheck, pnpm build (so TS/ESLint/Next build failures get caught). .github/workflows/ci.yml + - CI runs Go unit tests for agents/agentd (go test ./...). .github/workflows/ci.yml + - CI also does Docker builds + CodeQL + dependency review. + What we don’t have + - No unit/integration test runner for the dashboard/control-plane packages (they don’t even define a test script), so pnpm test + doesn’t exercise UI logic. + - No Playwright/Cypress-style e2e tests in CI for orchestrator/terminal interactions. Playwright exists, but it’s currently just + perf scripts under scripts/perf/, not regression coverage for these changes. + If you want, I can add real regression coverage next (either Vitest unit tests for the orchestrator store + extracted terminal + gesture logic, or Playwright smoke tests + a CI job). › Summarize recent commits - ? for shortcuts 45% context left + ? for shortcuts 60% context left ``` @@ -48,42 +46,27 @@ This file provides context for AI assistants (Claude Code, Cursor, etc.) about r | Hash | Message | Date | Author | |------|---------|------|--------| -| `9d4e0a0` | Merge pull request #44 from cvsloane/fix/codex-usage-buckets | 2026-02-13 | Chris Sloane | -| `691109b` | fix: codex-usage prefer non-spark limit bucket | 2026-02-13 | cvsloane | -| `91bc7f0` | fix: prefer non-spark codex usage + show spark breakdown | 2026-02-13 | cvsloane | -| `ffd866c` | Merge pull request #43 from cvsloane/fix/streaming-migrations | 2026-02-07 | Chris Sloane | -| `2b1a557` | fix: restore agent streaming by fixing migrations | 2026-02-07 | cvsloane | +| `877c499` | Merge pull request #49 from cvsloane/chore/tasks-note-followups | 2026-02-14 | Chris Sloane | +| `c26c10c` | chore: update task note with follow-up improvements | 2026-02-14 | cvsloane | +| `d4169e2` | Merge pull request #48 from cvsloane/fix/mobile-terminal-axis-lock | 2026-02-14 | Chris Sloane | +| `80f39d8` | fix: smooth mobile terminal gesture axis lock | 2026-02-14 | cvsloane | +| `949753e` | Merge pull request #47 from cvsloane/fix/fullscreen-terminal-orchestrator-ws | 2026-02-14 | Chris Sloane | --- ## Files Changed Recently -- `agents/agentd/cmd/agentd/main.go` -- `agents/agentd/cmd/agentd/main_usage_parse_test.go` -- `agents/agentd/go.mod` -- `agents/agentd/go.sum` -- `agents/agentd/internal/console/stream.go` -- `agents/agentd/internal/queue/queue.go` -- `apps/dashboard/src/app/(dashboard)/hosts/page.tsx` -- `apps/dashboard/src/app/(dashboard)/orchestrator/OrchestratorPageClient.tsx` -- `apps/dashboard/src/app/(dashboard)/page.tsx` - `apps/dashboard/src/app/(dashboard)/sessions/[id]/page.tsx` -- `apps/dashboard/src/app/(dashboard)/sessions/SessionsPageClient.tsx` -- `apps/dashboard/src/components/analytics/ProviderUtilization.tsx` -- `apps/dashboard/src/components/BulkActionToolbar.tsx` -- `apps/dashboard/src/components/SessionCard.tsx` -- `apps/dashboard/src/components/SessionList.tsx` -- `apps/dashboard/src/components/VoiceInputButton.tsx` -- `apps/dashboard/src/hooks/useOrchestratorSummaries.ts` -- `apps/dashboard/src/hooks/useTerminateSession.ts` -- `apps/dashboard/src/lib/api.ts` -- `apps/dashboard/src/lib/utils.ts` +- `apps/dashboard/src/components/orchestrator/OrchestratorButton.tsx` +- `apps/dashboard/src/components/TerminalView.tsx` +- `project_status.md` +- `tasks/2026-02-14-mobile-ux-orchestrator-refactor.md` --- ## Branch Info -- **Generated From Branch**: `chore/project-status` -- **Last Commit**: 2026-02-13T14:16:53-05:00 +- **Current Branch**: `main` +- **Last Commit**: 2026-02-14T12:00:47-05:00 From 956de1b0e22f118098a429e2159bf30d7ec6d85f Mon Sep 17 00:00:00 2001 From: cvsloane Date: Mon, 16 Feb 2026 10:46:07 -0500 Subject: [PATCH 2/6] feat(metrics): add /metrics and reliability inventory --- agents/agentd/go.mod | 16 ++- agents/agentd/go.sum | 46 +++++++- agents/agentd/internal/metrics/metrics.go | 101 ++++++++++++++++++ agents/agentd/internal/providers/claude.go | 2 + agents/agentd/internal/ws/client.go | 44 +++++--- docs/README.md | 1 + docs/agentd.md | 7 ++ docs/operations.md | 1 + docs/reliability-primitives.md | 60 +++++++++++ pnpm-lock.yaml | 41 ++++++- services/control-plane/.env.example | 6 ++ services/control-plane/package.json | 1 + services/control-plane/src/config.ts | 1 + services/control-plane/src/index.ts | 4 +- services/control-plane/src/metrics.ts | 39 +++++++ services/control-plane/src/routes/metrics.ts | 35 ++++++ .../control-plane/src/services/clawdbot.ts | 68 ++++++++++-- 17 files changed, 436 insertions(+), 37 deletions(-) create mode 100644 agents/agentd/internal/metrics/metrics.go create mode 100644 docs/reliability-primitives.md create mode 100644 services/control-plane/src/metrics.ts create mode 100644 services/control-plane/src/routes/metrics.ts diff --git a/agents/agentd/go.mod b/agents/agentd/go.mod index db85372c..63692bad 100644 --- a/agents/agentd/go.mod +++ b/agents/agentd/go.mod @@ -1,13 +1,25 @@ module github.com/agent-command/agentd -go 1.22 +go 1.23.0 require ( github.com/creack/pty v1.1.24 github.com/fsnotify/fsnotify v1.7.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 + github.com/prometheus/client_golang v1.23.2 gopkg.in/yaml.v3 v3.0.1 ) -require golang.org/x/sys v0.4.0 // indirect +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/sys v0.35.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect +) diff --git a/agents/agentd/go.sum b/agents/agentd/go.sum index 457677c9..9112cdc7 100644 --- a/agents/agentd/go.sum +++ b/agents/agentd/go.sum @@ -1,14 +1,54 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/agents/agentd/internal/metrics/metrics.go b/agents/agentd/internal/metrics/metrics.go new file mode 100644 index 00000000..69d9e729 --- /dev/null +++ b/agents/agentd/internal/metrics/metrics.go @@ -0,0 +1,101 @@ +package metrics + +import ( + "net/http" + "sync" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var once sync.Once + +var ( + wsConnected = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "agentd_ws_connected", + Help: "Whether agentd is currently connected to the control plane (1=yes, 0=no).", + }) + + wsReconnecting = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "agentd_ws_reconnecting", + Help: "Whether agentd is currently in a reconnect loop (1=yes, 0=no).", + }) + + wsReconnectAttemptsTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "agentd_ws_reconnect_attempts_total", + Help: "Total WebSocket reconnect attempts to the control plane.", + }) + + wsReconnectFailuresTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "agentd_ws_reconnect_failures_total", + Help: "Total WebSocket reconnect failures to the control plane.", + }) + + wsReconnectSuccessTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "agentd_ws_reconnect_success_total", + Help: "Total WebSocket reconnect successes to the control plane.", + }) + + wsReconnectBackoffSeconds = prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "agentd_ws_reconnect_backoff_seconds", + Help: "Backoff delay (sleep) used before each reconnect attempt.", + Buckets: []float64{0.25, 0.5, 1, 2, 5, 10, 30, 60}, + }) +) + +func initOnce() { + once.Do(func() { + prometheus.MustRegister( + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + wsConnected, + wsReconnecting, + wsReconnectAttemptsTotal, + wsReconnectFailuresTotal, + wsReconnectSuccessTotal, + wsReconnectBackoffSeconds, + ) + }) +} + +func Handler() http.Handler { + initOnce() + return promhttp.Handler() +} + +func SetWSConnected(connected bool) { + initOnce() + if connected { + wsConnected.Set(1) + return + } + wsConnected.Set(0) +} + +func SetWSReconnecting(reconnecting bool) { + initOnce() + if reconnecting { + wsReconnecting.Set(1) + return + } + wsReconnecting.Set(0) +} + +func RecordWSReconnectAttempt(delayMs int) { + initOnce() + wsReconnectAttemptsTotal.Inc() + if delayMs > 0 { + wsReconnectBackoffSeconds.Observe(float64(delayMs) / 1000.0) + } +} + +func RecordWSReconnectFailure() { + initOnce() + wsReconnectFailuresTotal.Inc() +} + +func RecordWSReconnectSuccess() { + initOnce() + wsReconnectSuccessTotal.Inc() +} diff --git a/agents/agentd/internal/providers/claude.go b/agents/agentd/internal/providers/claude.go index 211ed4de..5991c2e7 100644 --- a/agents/agentd/internal/providers/claude.go +++ b/agents/agentd/internal/providers/claude.go @@ -11,6 +11,7 @@ import ( "time" "github.com/agent-command/agentd/internal/config" + "github.com/agent-command/agentd/internal/metrics" "github.com/google/uuid" ) @@ -63,6 +64,7 @@ func (p *ClaudeProvider) Start() error { mux := http.NewServeMux() mux.HandleFunc("/v1/hooks/claude", p.handleHook) mux.HandleFunc("/v1/hooks/codex", p.handleCodexHook) + mux.Handle("/metrics", metrics.Handler()) p.server = &http.Server{ Addr: p.cfg.HooksHTTPListen, diff --git a/agents/agentd/internal/ws/client.go b/agents/agentd/internal/ws/client.go index ec3208cc..74111def 100644 --- a/agents/agentd/internal/ws/client.go +++ b/agents/agentd/internal/ws/client.go @@ -10,27 +10,28 @@ import ( "sync/atomic" "time" - "github.com/gorilla/websocket" + "github.com/agent-command/agentd/internal/metrics" "github.com/agent-command/agentd/internal/queue" + "github.com/gorilla/websocket" ) type MessageHandler func(msgType string, payload json.RawMessage) type Client struct { - url string - token string - hostID string - backoff []int - conn *websocket.Conn - mu sync.Mutex - seq atomic.Int64 - lastAckedSeq int64 - onMessage MessageHandler - done chan struct{} - reconnecting bool - queue *queue.Queue - stateDir string - onConnect func() + url string + token string + hostID string + backoff []int + conn *websocket.Conn + mu sync.Mutex + seq atomic.Int64 + lastAckedSeq int64 + onMessage MessageHandler + done chan struct{} + reconnecting bool + queue *queue.Queue + stateDir string + onConnect func() } func NewClient(url, token, hostID string, backoff []int) *Client { @@ -76,6 +77,8 @@ func (c *Client) Connect() error { c.conn = conn c.reconnecting = false + metrics.SetWSConnected(true) + metrics.SetWSReconnecting(false) // Start reader goroutine go c.reader() @@ -95,6 +98,8 @@ func (c *Client) reader() { } c.mu.Unlock() + metrics.SetWSConnected(false) + // Attempt reconnection c.reconnect() }() @@ -174,6 +179,9 @@ func (c *Client) reconnect() { c.reconnecting = true c.mu.Unlock() + metrics.SetWSReconnecting(true) + defer metrics.SetWSReconnecting(false) + for i, delay := range c.backoff { select { case <-c.done: @@ -181,12 +189,15 @@ func (c *Client) reconnect() { case <-time.After(time.Duration(delay) * time.Millisecond): } + metrics.RecordWSReconnectAttempt(delay) log.Printf("Reconnection attempt %d/%d", i+1, len(c.backoff)) if err := c.Connect(); err == nil { + metrics.RecordWSReconnectSuccess() log.Printf("Reconnected successfully") return } + metrics.RecordWSReconnectFailure() } // Keep trying with max backoff @@ -198,10 +209,13 @@ func (c *Client) reconnect() { case <-time.After(time.Duration(maxDelay) * time.Millisecond): } + metrics.RecordWSReconnectAttempt(maxDelay) if err := c.Connect(); err == nil { + metrics.RecordWSReconnectSuccess() log.Printf("Reconnected successfully") return } + metrics.RecordWSReconnectFailure() } } diff --git a/docs/README.md b/docs/README.md index 7c66318c..70b09045 100644 --- a/docs/README.md +++ b/docs/README.md @@ -35,5 +35,6 @@ Reference: Operations: - [Operations](operations.md) +- [Reliability Primitives](reliability-primitives.md) - [Troubleshooting](troubleshooting.md) - [Implementation Notes](implementation.md) diff --git a/docs/agentd.md b/docs/agentd.md index de10ffe8..03c3bccc 100644 --- a/docs/agentd.md +++ b/docs/agentd.md @@ -54,6 +54,13 @@ Disable these if you want read only mode on a host. agentd can handle provider specific hooks and usage parsing. See [Provider Hooks](hooks.md) for setup details. +### Metrics + +agentd exposes Prometheus metrics at `GET /metrics` on the same HTTP listener as provider hooks (`providers.claude.hooks_http_listen`). + +- Default: `http://127.0.0.1:7777/metrics` +- If you bind hooks to a non-loopback address, treat `/metrics` as sensitive (firewall or reverse proxy auth). + ### Provider usage polling agentd can run periodic commands to pull usage data from provider CLIs or APIs diff --git a/docs/operations.md b/docs/operations.md index e5427828..c3fddf02 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -10,6 +10,7 @@ This guide covers operational best practices for running Agent Commander. ## Monitoring - Watch `/health` for control plane status. +- Scrape `/metrics` for operational counters (reconnect/backoff, Clawdbot rate-limit decisions). If `METRICS_TOKEN` is set on the control plane, Prometheus must send `Authorization: Bearer ` (or `x-metrics-token`). - Monitor database connections and disk usage. - Track agentd logs for reconnect loops or tmux errors. diff --git a/docs/reliability-primitives.md b/docs/reliability-primitives.md new file mode 100644 index 00000000..c7558931 --- /dev/null +++ b/docs/reliability-primitives.md @@ -0,0 +1,60 @@ +# Reliability Primitives + +This doc inventories how **retry/backoff**, **rate limiting**, and **ops metrics** currently work in Agent Command, and proposes a migration path toward consistent, observable behavior. + +## Inventory (Current State) + +### Retry + Backoff + +**agentd (Go)** +- WebSocket reconnect loop: `agents/agentd/internal/ws/client.go` + - Policy: fixed backoff schedule from config `control_plane.reconnect_backoff_ms` (default `[250, 500, 1000, 2000, 5000]` ms) and then retries forever at the max delay. + - No jitter. + - No per-attempt timeout (relies on the underlying dial behavior). +- Ad-hoc “retry once” for Claude usage command parsing: `agents/agentd/cmd/agentd/main.go` (`maybeRetryClaudeUsage`) + - If output looks incomplete, re-runs the command using `script -c` to force a TTY. + - No backoff. + +**control-plane (TypeScript)** +- No shared retry helper (outbound HTTP calls generally rely on per-call timeouts). +- One “retry for race” pattern exists in group creation logic (check-then-recheck on unique collision): `services/control-plane/src/routes/groups.ts` and `services/control-plane/src/db/index.ts`. + +### Rate Limiting + +**control-plane (TypeScript)** +- In-memory per-user throttle for Clawdbot notifications: `services/control-plane/src/services/clawdbot.ts` + - Default: `maxPerHour = 30` with additional dedupe + session cooldown rules. + - Not distributed-safe (assumes a single control-plane instance for correct enforcement). + +### Metrics / Observability + +**Before 2026-02-16:** operational metrics (Prometheus-style) were not exposed; only session analytics existed in Postgres (`migrations/009_session_analytics.sql`). + +**After 2026-02-16:** operational `/metrics` endpoints exist: +- control-plane: `GET /metrics` (token optional via `METRICS_TOKEN`) +- agentd: `GET /metrics` served from the hooks HTTP server (same listen address as `providers.claude.hooks_http_listen`) + +## Standard Primitives (Recommended) + +### Go (agentd) +1. **Retry/backoff standard:** exponential backoff with jitter and a hard cap. +2. **Timeout standard:** every outbound operation should be bounded by context deadlines. +3. **Metrics standard:** Prometheus counters/histograms for reconnect attempts, failures, and backoff delays. + +### TypeScript (control-plane) +1. **Retry/backoff standard:** one shared helper (jitter, caps, per-attempt timeout, and explicit “non-retryable” classification). +2. **Rate limiting standard:** if control-plane is ever scaled horizontally, move throttles to Redis-backed primitives (in-memory is fine only when single-instance is guaranteed). +3. **Metrics standard:** Prometheus counters/histograms for decisioning (allowed/blocked + reason) and key outbound call failures. + +## Migration Plan (Incremental) + +1. **(Done) Add operational metrics endpoints and key counters** + - control-plane `/metrics` + Clawdbot allow/deny reasons + - agentd `/metrics` + reconnect/backoff counters +2. **Centralize retry/backoff** + - Introduce a small `retry` helper in each language (repo-local), then migrate high-value call sites first. +3. **Rate limiting hardening (only if/when needed)** + - If control-plane becomes multi-instance: add Redis and migrate Clawdbot throttle to distributed-safe limits. +4. **Dashboards/alerts** + - Alert on reconnect storms, reconnect failure rates, and sustained Clawdbot rate-limit blocks. + diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5e779066..c18a501c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -100,10 +100,10 @@ importers: version: 0.460.0(react@18.3.1) next: specifier: ^14.2.0 - version: 14.2.35(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + version: 14.2.35(@opentelemetry/api@1.9.0)(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) next-auth: specifier: ^4.24.0 - version: 4.24.13(next@14.2.35(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + version: 4.24.13(next@14.2.35(@opentelemetry/api@1.9.0)(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1) react: specifier: ^18.3.0 version: 18.3.1 @@ -214,6 +214,9 @@ importers: pino-pretty: specifier: ^10.3.1 version: 10.3.1 + prom-client: + specifier: ^15.1.3 + version: 15.1.3 ulid: specifier: ^2.3.0 version: 2.4.0 @@ -619,6 +622,10 @@ packages: resolution: {integrity: sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==} engines: {node: '>=12.4.0'} + '@opentelemetry/api@1.9.0': + resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} + engines: {node: '>=8.0.0'} + '@panva/hkdf@1.2.1': resolution: {integrity: sha512-6oclG6Y3PiDFcoyk8srjLfVKyMfVCKJ27JwNPViuXziFpmdz+MZnZN/aKY0JGXgYuO/VghU0jcOAZgWXZ1Dmrw==} @@ -1560,6 +1567,9 @@ packages: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} + bintrees@1.0.2: + resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} + brace-expansion@1.1.12: resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} @@ -3032,6 +3042,10 @@ packages: resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} engines: {node: '>= 0.6.0'} + prom-client@15.1.3: + resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==} + engines: {node: ^16 || ^18 || >=20} + promise-worker-transferable@1.0.4: resolution: {integrity: sha512-bN+0ehEnrXfxV2ZQvU2PetO0n4gqBD4ulq3MI1WOPLgr7/Mg9yRQkX5+0v1vagr74ZTsl7XtzlaYDo2EuCeYJw==} @@ -3434,6 +3448,9 @@ packages: engines: {node: '>=14.0.0'} hasBin: true + tdigest@0.1.2: + resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} + text-table@0.2.0: resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==} @@ -4120,6 +4137,8 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} + '@opentelemetry/api@1.9.0': {} + '@panva/hkdf@1.2.1': {} '@pinojs/redact@0.4.0': {} @@ -5099,6 +5118,8 @@ snapshots: binary-extensions@2.3.0: {} + bintrees@1.0.2: {} + brace-expansion@1.1.12: dependencies: balanced-match: 1.0.2 @@ -6614,13 +6635,13 @@ snapshots: natural-compare@1.4.0: {} - next-auth@4.24.13(next@14.2.35(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + next-auth@4.24.13(next@14.2.35(@opentelemetry/api@1.9.0)(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: '@babel/runtime': 7.28.6 '@panva/hkdf': 1.2.1 cookie: 0.7.2 jose: 4.15.9 - next: 14.2.35(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + next: 14.2.35(@opentelemetry/api@1.9.0)(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) oauth: 0.9.15 openid-client: 5.7.1 preact: 10.28.2 @@ -6629,7 +6650,7 @@ snapshots: react-dom: 18.3.1(react@18.3.1) uuid: 8.3.2 - next@14.2.35(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + next@14.2.35(@opentelemetry/api@1.9.0)(@playwright/test@1.57.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: '@next/env': 14.2.35 '@swc/helpers': 0.5.5 @@ -6650,6 +6671,7 @@ snapshots: '@next/swc-win32-arm64-msvc': 14.2.33 '@next/swc-win32-ia32-msvc': 14.2.33 '@next/swc-win32-x64-msvc': 14.2.33 + '@opentelemetry/api': 1.9.0 '@playwright/test': 1.57.0 transitivePeerDependencies: - '@babel/core' @@ -6960,6 +6982,11 @@ snapshots: process@0.11.10: {} + prom-client@15.1.3: + dependencies: + '@opentelemetry/api': 1.9.0 + tdigest: 0.1.2 + promise-worker-transferable@1.0.4: dependencies: is-promise: 2.2.2 @@ -7456,6 +7483,10 @@ snapshots: - tsx - yaml + tdigest@0.1.2: + dependencies: + bintrees: 1.0.2 + text-table@0.2.0: {} thenify-all@1.6.0: diff --git a/services/control-plane/.env.example b/services/control-plane/.env.example index 82ce1017..b8ccba0a 100644 --- a/services/control-plane/.env.example +++ b/services/control-plane/.env.example @@ -14,6 +14,12 @@ DATABASE_URL= HOST=0.0.0.0 PORT=8080 +# Optional: protect `GET /metrics` with a bearer token. +# If set, provide either: +# - `Authorization: Bearer ` +# - `x-metrics-token: ` +METRICS_TOKEN= + # ----------------------------------------------------------------------------- # SECURITY # ----------------------------------------------------------------------------- diff --git a/services/control-plane/package.json b/services/control-plane/package.json index 024461a9..25d50f67 100644 --- a/services/control-plane/package.json +++ b/services/control-plane/package.json @@ -25,6 +25,7 @@ "openai": "^6.16.0", "pg": "^8.13.0", "pino-pretty": "^10.3.1", + "prom-client": "^15.1.3", "ulid": "^2.3.0", "zod": "^3.23.0" }, diff --git a/services/control-plane/src/config.ts b/services/control-plane/src/config.ts index 71464ee9..939b81cf 100644 --- a/services/control-plane/src/config.ts +++ b/services/control-plane/src/config.ts @@ -6,6 +6,7 @@ const ConfigSchema = z.object({ HOST: z.string().default('0.0.0.0'), PORT: z.coerce.number().default(8080), JWT_SECRET: z.string().min(16), + METRICS_TOKEN: z.string().min(1).optional(), TAILNET_DOMAIN: z.string().optional(), OPENAI_API_KEY: z.string().optional(), OPENAI_MODEL: z.string().default('gpt-4o-mini'), diff --git a/services/control-plane/src/index.ts b/services/control-plane/src/index.ts index 136bdf82..a3fc9ef4 100644 --- a/services/control-plane/src/index.ts +++ b/services/control-plane/src/index.ts @@ -20,6 +20,7 @@ import { registerProjectRoutes } from './routes/projects.js'; import { registerSummaryRoutes } from './routes/summaries.js'; import { registerSettingsRoutes } from './routes/settings.js'; import { registerNotificationRoutes } from './routes/notifications.js'; +import { registerMetricsRoutes } from './routes/metrics.js'; import { pubsub } from './services/pubsub.js'; import { verifyRequestToken } from './auth/verify.js'; @@ -62,7 +63,7 @@ async function start(): Promise { // Auth for REST (skip health + WS routes that handle their own auth) app.addHook('onRequest', async (request, reply) => { const url = request.url; - if (url.startsWith('/health') || url.startsWith('/v1/agent/connect') || url.startsWith('/v1/ui/stream') || url.startsWith('/v1/ui/terminal') || url.startsWith('/v1/voice/transcribe')) { + if (url.startsWith('/health') || url.startsWith('/metrics') || url.startsWith('/v1/agent/connect') || url.startsWith('/v1/ui/stream') || url.startsWith('/v1/ui/terminal') || url.startsWith('/v1/voice/transcribe')) { return; } @@ -103,6 +104,7 @@ async function start(): Promise { registerSummaryRoutes(app); registerSettingsRoutes(app); registerNotificationRoutes(app); + registerMetricsRoutes(app); // Health check endpoint app.get('/health', async () => { diff --git a/services/control-plane/src/metrics.ts b/services/control-plane/src/metrics.ts new file mode 100644 index 00000000..0cc95c2f --- /dev/null +++ b/services/control-plane/src/metrics.ts @@ -0,0 +1,39 @@ +import client from 'prom-client'; + +export const registry = new client.Registry(); +registry.setDefaultLabels({ app: 'agent-command-control-plane' }); + +client.collectDefaultMetrics({ register: registry }); + +export type ClawdbotDecision = 'allowed' | 'blocked'; +export type ClawdbotReason = + | 'allowed' + | 'provider_disabled' + | 'actionable_only' + | 'event_disabled' + | 'rate_limit' + | 'approval_dedup' + | 'dedupe_key' + | 'session_cooldown'; + +export const clawdbotNotificationDecisionsTotal = new client.Counter({ + name: 'agent_command_clawdbot_notification_decisions_total', + help: 'Count of Clawdbot notification decisions (allowed/blocked) and reasons.', + labelNames: ['decision', 'reason', 'event_type', 'provider'] as const, + registers: [registry], +}); + +export function recordClawdbotNotificationDecision(params: { + decision: ClawdbotDecision; + reason: ClawdbotReason; + eventType: string; + provider: string | null; +}): void { + clawdbotNotificationDecisionsTotal.inc({ + decision: params.decision, + reason: params.reason, + event_type: params.eventType, + provider: params.provider ?? 'none', + }); +} + diff --git a/services/control-plane/src/routes/metrics.ts b/services/control-plane/src/routes/metrics.ts new file mode 100644 index 00000000..ecea7995 --- /dev/null +++ b/services/control-plane/src/routes/metrics.ts @@ -0,0 +1,35 @@ +import type { FastifyInstance } from 'fastify'; +import { config } from '../config.js'; +import { registry } from '../metrics.js'; + +function getBearerToken(authHeader: string | undefined): string | null { + if (!authHeader) return null; + if (!authHeader.startsWith('Bearer ')) return null; + return authHeader.slice('Bearer '.length); +} + +function getHeaderToken(value: unknown): string | null { + if (typeof value === 'string') return value; + if (Array.isArray(value) && typeof value[0] === 'string') return value[0]; + return null; +} + +export function registerMetricsRoutes(app: FastifyInstance): void { + app.get('/metrics', async (request, reply) => { + const expected = config.METRICS_TOKEN; + if (expected) { + const bearer = getBearerToken(request.headers.authorization); + const direct = getHeaderToken(request.headers['x-metrics-token']); + const provided = bearer ?? direct; + if (provided !== expected) { + return reply.status(401).send('Unauthorized\n'); + } + } + + const body = await registry.metrics(); + reply.header('Content-Type', registry.contentType); + reply.header('Cache-Control', 'no-store'); + return body; + }); +} + diff --git a/services/control-plane/src/services/clawdbot.ts b/services/control-plane/src/services/clawdbot.ts index 609ed264..2e975aba 100644 --- a/services/control-plane/src/services/clawdbot.ts +++ b/services/control-plane/src/services/clawdbot.ts @@ -1,6 +1,7 @@ import type { Session, Approval } from '@agent-command/schema'; import * as db from '../db/index.js'; import { pool } from '../db/index.js'; +import { recordClawdbotNotificationDecision, type ClawdbotReason } from '../metrics.js'; interface ClawdbotThrottleSettings { maxPerHour: number; @@ -32,6 +33,12 @@ interface RateLimitEntry { windowStart: number; } +type NotifyBlockReason = Exclude; + +type NotifyDecision = + | { allowed: true } + | { allowed: false; reason: { code: NotifyBlockReason; detail: string } }; + // Default events: only critical ones enabled const DEFAULT_CLAWDBOT_EVENTS: Record = { approvals: true, @@ -127,7 +134,7 @@ class ClawdbotNotifier { approvalId?: string; status?: string; } = {} - ): { allowed: boolean; reason?: string } { + ): NotifyDecision { const { isActionable, sessionId, approvalId, status } = options; const throttle = config.throttle ?? DEFAULT_THROTTLE; const actionable = isActionable !== false; @@ -135,23 +142,35 @@ class ClawdbotNotifier { // 1. Check actionableOnly filter if (actionableOnly && !actionable) { - return { allowed: false, reason: 'actionableOnly filter (not actionable)' }; + return { + allowed: false, + reason: { code: 'actionable_only', detail: 'actionableOnly filter (not actionable)' }, + }; } // 2. Check event type filter const eventEnabled = config.events[eventType] ?? DEFAULT_CLAWDBOT_EVENTS[eventType] ?? true; if (!eventEnabled) { - return { allowed: false, reason: `event type '${eventType}' disabled` }; + return { + allowed: false, + reason: { code: 'event_disabled', detail: `event type '${eventType}' disabled` }, + }; } // 3. Check rate limit (non-mutating) if (!this.canSend(userId, throttle.maxPerHour)) { - return { allowed: false, reason: 'rate limit exceeded' }; + return { + allowed: false, + reason: { code: 'rate_limit', detail: 'rate limit exceeded' }, + }; } // 4. Approval-specific deduplication if (approvalId && this.approvalNotified.has(this.approvalKey(userId, approvalId))) { - return { allowed: false, reason: `approval ${approvalId} already notified` }; + return { + allowed: false, + reason: { code: 'approval_dedup', detail: `approval ${approvalId} already notified` }, + }; } // 5. Strong deduplication by dedupeKey @@ -160,7 +179,10 @@ class ClawdbotNotifier { if (lastNotified) { const dedupeWindowMs = 5 * 60 * 1000; // 5 minutes if (Date.now() - lastNotified < dedupeWindowMs) { - return { allowed: false, reason: `duplicate notification (key: ${dedupeKey})` }; + return { + allowed: false, + reason: { code: 'dedupe_key', detail: `duplicate notification (key: ${dedupeKey})` }, + }; } } @@ -168,7 +190,10 @@ class ClawdbotNotifier { if (sessionId && !actionable) { const sessionLastTime = this.sessionLastNotified.get(this.sessionKey(userId, sessionId)); if (sessionLastTime && Date.now() - sessionLastTime < throttle.sessionCooldownMs) { - return { allowed: false, reason: `session ${sessionId} cooldown active` }; + return { + allowed: false, + reason: { code: 'session_cooldown', detail: `session ${sessionId} cooldown active` }, + }; } } @@ -217,16 +242,37 @@ class ClawdbotNotifier { // Check provider filter if (provider) { const providerEnabled = config.providers[provider] ?? true; - if (!providerEnabled) return; + if (!providerEnabled) { + recordClawdbotNotificationDecision({ + decision: 'blocked', + reason: 'provider_disabled', + eventType, + provider, + }); + return; + } } // Run shouldNotify checks - const { allowed, reason } = this.shouldNotify(userId, eventType, config, options); - if (!allowed) { - console.log(`[clawdbot] Notification blocked for user ${userId}: ${reason}`); + const decision = this.shouldNotify(userId, eventType, config, options); + if (!decision.allowed) { + recordClawdbotNotificationDecision({ + decision: 'blocked', + reason: decision.reason.code, + eventType, + provider, + }); + console.log(`[clawdbot] Notification blocked for user ${userId}: ${decision.reason.detail}`); return; } + recordClawdbotNotificationDecision({ + decision: 'allowed', + reason: 'allowed', + eventType, + provider, + }); + const throttle = config.throttle ?? DEFAULT_THROTTLE; // Consume rate limit only after passing all checks From eaefe89b8f5dbf33efbb676b77f2736cd55d0e22 Mon Sep 17 00:00:00 2001 From: cvsloane Date: Mon, 16 Feb 2026 10:51:48 -0500 Subject: [PATCH 3/6] fix(agentd): pin prometheus client to Go 1.22 --- agents/agentd/go.mod | 2 +- agents/agentd/go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/agents/agentd/go.mod b/agents/agentd/go.mod index 63692bad..a3c070c3 100644 --- a/agents/agentd/go.mod +++ b/agents/agentd/go.mod @@ -7,7 +7,7 @@ require ( github.com/fsnotify/fsnotify v1.7.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 - github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_golang v1.22.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/agents/agentd/go.sum b/agents/agentd/go.sum index 9112cdc7..1f65f6cd 100644 --- a/agents/agentd/go.sum +++ b/agents/agentd/go.sum @@ -27,8 +27,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= -github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= @@ -39,8 +39,6 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= From 677fadf21bfde60ce73de8648673434cdd4d54ae Mon Sep 17 00:00:00 2001 From: cvsloane Date: Mon, 16 Feb 2026 10:58:07 -0500 Subject: [PATCH 4/6] fix(agentd): keep deps Go 1.22 compatible --- agents/agentd/go.mod | 11 +++++------ agents/agentd/go.sum | 22 ++++++++++------------ 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/agents/agentd/go.mod b/agents/agentd/go.mod index a3c070c3..a63160a4 100644 --- a/agents/agentd/go.mod +++ b/agents/agentd/go.mod @@ -1,6 +1,6 @@ module github.com/agent-command/agentd -go 1.23.0 +go 1.22.0 require ( github.com/creack/pty v1.1.24 @@ -17,9 +17,8 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.66.1 // indirect - github.com/prometheus/procfs v0.16.1 // indirect - go.yaml.in/yaml/v2 v2.4.2 // indirect - golang.org/x/sys v0.35.0 // indirect - google.golang.org/protobuf v1.36.8 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + golang.org/x/sys v0.30.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect ) diff --git a/agents/agentd/go.sum b/agents/agentd/go.sum index 1f65f6cd..059fbf25 100644 --- a/agents/agentd/go.sum +++ b/agents/agentd/go.sum @@ -31,20 +31,18 @@ github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/ github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= -github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= -github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= -github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= -google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From bb433676a26ec5405dbae758ef2c38f7eff55ded Mon Sep 17 00:00:00 2001 From: cvsloane Date: Mon, 16 Feb 2026 11:02:03 -0500 Subject: [PATCH 5/6] chore: update project status --- project_status.md | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/project_status.md b/project_status.md index 0196bb9c..a9b0007b 100644 --- a/project_status.md +++ b/project_status.md @@ -1,7 +1,7 @@ # Project Status - agent-command > **Auto-generated by [project-status agent](https://github.com/cvsloane/open-agents)** -> Last updated: 2026-02-15T16:02:03.149Z +> Last updated: 2026-02-16T16:02:03.813Z ## Purpose @@ -18,7 +18,6 @@ This file provides context for AI assistants (Claude Code, Cursor, etc.) about r **Recent Terminal Output**: ``` - I should mention a few manual checks we can perform: opening the session page, toggling fullscreen, checking for scroll lock, and testing scrolling on mobile. We can also verify if the orchestrator opens correctly. I'll keep this part minimal to avoid overwhelming the user. Now, I’ll craft the final response to include these manual checks along with the automated testing discussion we've had earlier. @@ -35,6 +34,7 @@ This file provides context for AI assistants (Claude Code, Cursor, etc.) about r perf scripts under scripts/perf/, not regression coverage for these changes. If you want, I can add real regression coverage next (either Vitest unit tests for the orchestrator store + extracted terminal gesture logic, or Playwright smoke tests + a CI job). +⚠ Heads up, you have less than 25% of your weekly limit left. Run /status for a breakdown. › Summarize recent commits ? for shortcuts 60% context left ``` @@ -46,27 +46,42 @@ This file provides context for AI assistants (Claude Code, Cursor, etc.) about r | Hash | Message | Date | Author | |------|---------|------|--------| +| `677fadf` | fix(agentd): keep deps Go 1.22 compatible | 2026-02-16 | cvsloane | +| `eaefe89` | fix(agentd): pin prometheus client to Go 1.22 | 2026-02-16 | cvsloane | +| `956de1b` | feat(metrics): add /metrics and reliability inventory | 2026-02-16 | cvsloane | +| `38bddd7` | chore: update project status | 2026-02-15 | cvsloane | | `877c499` | Merge pull request #49 from cvsloane/chore/tasks-note-followups | 2026-02-14 | Chris Sloane | -| `c26c10c` | chore: update task note with follow-up improvements | 2026-02-14 | cvsloane | -| `d4169e2` | Merge pull request #48 from cvsloane/fix/mobile-terminal-axis-lock | 2026-02-14 | Chris Sloane | -| `80f39d8` | fix: smooth mobile terminal gesture axis lock | 2026-02-14 | cvsloane | -| `949753e` | Merge pull request #47 from cvsloane/fix/fullscreen-terminal-orchestrator-ws | 2026-02-14 | Chris Sloane | --- ## Files Changed Recently +- `agents/agentd/go.mod` +- `agents/agentd/go.sum` +- `agents/agentd/internal/metrics/metrics.go` +- `agents/agentd/internal/providers/claude.go` +- `agents/agentd/internal/ws/client.go` - `apps/dashboard/src/app/(dashboard)/sessions/[id]/page.tsx` - `apps/dashboard/src/components/orchestrator/OrchestratorButton.tsx` - `apps/dashboard/src/components/TerminalView.tsx` +- `docs/agentd.md` +- `docs/operations.md` +- `docs/README.md` +- `docs/reliability-primitives.md` +- `pnpm-lock.yaml` - `project_status.md` -- `tasks/2026-02-14-mobile-ux-orchestrator-refactor.md` +- `services/control-plane/.env.example` +- `services/control-plane/package.json` +- `services/control-plane/src/config.ts` +- `services/control-plane/src/index.ts` +- `services/control-plane/src/metrics.ts` +- `services/control-plane/src/routes/metrics.ts` --- ## Branch Info -- **Current Branch**: `main` -- **Last Commit**: 2026-02-14T12:00:47-05:00 +- **Current Branch**: `feat/ops-metrics` +- **Last Commit**: 2026-02-16T10:58:07-05:00 From c4a18b547bf30ccb39f93967d314cbe61aef13cb Mon Sep 17 00:00:00 2001 From: cvsloane Date: Mon, 16 Feb 2026 11:04:34 -0500 Subject: [PATCH 6/6] fix(agentd): avoid duplicate Prometheus collector registration --- agents/agentd/internal/metrics/metrics.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/agents/agentd/internal/metrics/metrics.go b/agents/agentd/internal/metrics/metrics.go index 69d9e729..c2a27283 100644 --- a/agents/agentd/internal/metrics/metrics.go +++ b/agents/agentd/internal/metrics/metrics.go @@ -5,7 +5,6 @@ import ( "sync" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -47,8 +46,6 @@ var ( func initOnce() { once.Do(func() { prometheus.MustRegister( - collectors.NewGoCollector(), - collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), wsConnected, wsReconnecting, wsReconnectAttemptsTotal,