From c8b9dbc1068d9a16339fabf1fbc7c80699e34143 Mon Sep 17 00:00:00 2001 From: cvsloane Date: Wed, 22 Jul 2026 17:44:34 -0400 Subject: [PATCH 1/2] fix(terminal): prioritize pane focus acknowledgements --- agents/agentd/cmd/agentd/main.go | 2 +- agents/agentd/internal/ws/client.go | 5 +- agents/agentd/internal/ws/client_test.go | 47 +++++++++++++++++++ .../internal/ws/protocol_fixtures_test.go | 2 +- docs/operations.md | 7 ++- packages/ac-schema/src/message.ts | 5 +- packages/ac-schema/tests/terminal.test.ts | 2 +- services/control-plane/src/routes/terminal.ts | 28 ++++++----- services/control-plane/src/ws/agent.ts | 14 +++--- .../tests/agentWebSocket.test.ts | 39 +++++++++++++++ .../control-plane/tests/terminalRoute.test.ts | 12 +++-- tasks/lessons.md | 4 ++ .../protocol/terminal-navigation-result.json | 2 +- 13 files changed, 139 insertions(+), 30 deletions(-) diff --git a/agents/agentd/cmd/agentd/main.go b/agents/agentd/cmd/agentd/main.go index bad269fc..77f122c1 100644 --- a/agents/agentd/cmd/agentd/main.go +++ b/agents/agentd/cmd/agentd/main.go @@ -1775,7 +1775,7 @@ func (a *Agent) send(msgType string, payload any) error { var err error if a.sendMessage != nil { err = a.sendMessage(msgType, payload) - } else if msgType == protocol.TypeTmuxTopology { + } else if msgType == protocol.TypeTmuxTopology || msgType == protocol.TypeTerminalNavigationResult { err = a.wsClient.SendUnsequenced(msgType, payload) } else { err = a.wsClient.Send(msgType, payload) diff --git a/agents/agentd/internal/ws/client.go b/agents/agentd/internal/ws/client.go index 1c60bbe8..63627d30 100644 --- a/agents/agentd/internal/ws/client.go +++ b/agents/agentd/internal/ws/client.go @@ -366,9 +366,6 @@ func (c *Client) Send(msgType string, payload any) error { // participate in the durable agent sequence. It is used for full-state events // whose frozen contract omits seq and whose next snapshot supersedes the last. func (c *Client) SendUnsequenced(msgType string, payload any) error { - c.sendMu.Lock() - defer c.sendMu.Unlock() - payloadBytes, err := json.Marshal(payload) if err != nil { return fmt.Errorf("failed to marshal payload: %w", err) @@ -384,6 +381,8 @@ func (c *Client) SendUnsequenced(msgType string, payload any) error { return fmt.Errorf("failed to marshal %s envelope: %w", msgType, err) } + // c.mu serializes the actual WebSocket write. Deliberately avoid sendMu: + // unsequenced live responses must not wait behind durable queue fsync. c.mu.Lock() defer c.mu.Unlock() if c.conn == nil || !c.ready { diff --git a/agents/agentd/internal/ws/client_test.go b/agents/agentd/internal/ws/client_test.go index b5e08b05..fed5bfaa 100644 --- a/agents/agentd/internal/ws/client_test.go +++ b/agents/agentd/internal/ws/client_test.go @@ -116,6 +116,53 @@ func TestSendUnsequencedOmitsSequenceField(t *testing.T) { } } +func TestSendUnsequencedBypassesDurablePersistenceLane(t *testing.T) { + messages := make(chan receivedEnvelope, 1) + upgrader := websocket.Upgrader{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() + var message receivedEnvelope + if err := conn.ReadJSON(&message); err == nil { + messages <- message + } + })) + defer server.Close() + + client := NewClient(websocketURL(server), "token", "host", []int{1}) + if err := client.Connect(); err != nil { + t.Fatal(err) + } + defer client.Close() + if err := client.ResendQueued(); err != nil { + t.Fatal(err) + } + + client.sendMu.Lock() + sendDone := make(chan error, 1) + go func() { + sendDone <- client.SendUnsequenced("terminal.navigation_result", map[string]any{"ok": true}) + }() + + select { + case message := <-messages: + client.sendMu.Unlock() + if message.Type != "terminal.navigation_result" || message.Seq != 0 { + t.Fatalf("priority message=(%s,%d)", message.Type, message.Seq) + } + if err := <-sendDone; err != nil { + t.Fatal(err) + } + case <-time.After(100 * time.Millisecond): + client.sendMu.Unlock() + <-sendDone + t.Fatal("unsequenced message waited behind durable persistence lane") + } +} + func TestSequenceResumesAboveReloadedQueueMaximum(t *testing.T) { dir := t.TempDir() q, err := queue.NewQueue(dir, 100) diff --git a/agents/agentd/internal/ws/protocol_fixtures_test.go b/agents/agentd/internal/ws/protocol_fixtures_test.go index 9f899477..b57aa083 100644 --- a/agents/agentd/internal/ws/protocol_fixtures_test.go +++ b/agents/agentd/internal/ws/protocol_fixtures_test.go @@ -160,7 +160,7 @@ func fixtureMessageTarget(t *testing.T, messageType string, agentMessage bool) a case protocol.TypeTerminalOutput: return &protocol.AgentMessage[protocol.TerminalOutputPayload]{} case protocol.TypeTerminalNavigationResult: - return &protocol.AgentMessage[protocol.TerminalNavigationResultPayload]{} + return &protocol.ServerMessage[protocol.TerminalNavigationResultPayload]{} case protocol.TypeTerminalAudit: return &protocol.AgentMessage[protocol.TerminalAuditPayload]{} case protocol.TypeTmuxTopology: diff --git a/docs/operations.md b/docs/operations.md index ccf3160d..e1861c89 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -26,7 +26,12 @@ This guide covers operational best practices for running Agent Commander. - Run database migrations before restarting services. - Upgrade the dashboard and control plane together when possible. -- Upgrade agentd with the same release to keep schema in sync. +- For releases that change the live agent protocol, upgrade the control plane before + agentd; roll back agentd before rolling back the control plane. In particular, + control planes that support unsequenced `terminal.navigation_result` messages must + be live before an agentd version that emits them. +- Upgrade agentd with the same release after the control plane is healthy to keep the + fleet schema in sync. ## Secrets diff --git a/packages/ac-schema/src/message.ts b/packages/ac-schema/src/message.ts index 6b75c01c..35deb5ac 100644 --- a/packages/ac-schema/src/message.ts +++ b/packages/ac-schema/src/message.ts @@ -160,7 +160,10 @@ const TerminalNavigationResultPayloadSchema = z.intersection( ]) ); -export const TerminalNavigationResultMessageSchema = AgentMessageEnvelopeSchema.extend({ +// Viewer acknowledgements are live channel responses, not durable host events. +// Keeping them outside the durable cursor prevents queue persistence from +// delaying a browser-visible focus result. +export const TerminalNavigationResultMessageSchema = ServerMessageEnvelopeSchema.extend({ type: z.literal('terminal.navigation_result'), payload: TerminalNavigationResultPayloadSchema, }); diff --git a/packages/ac-schema/tests/terminal.test.ts b/packages/ac-schema/tests/terminal.test.ts index 693810ee..0212feba 100644 --- a/packages/ac-schema/tests/terminal.test.ts +++ b/packages/ac-schema/tests/terminal.test.ts @@ -78,7 +78,6 @@ describe('terminal protocol', () => { const result = TerminalNavigationResultMessageSchema.parse({ v: 1, - seq: 3, type: 'terminal.navigation_result', ts: '2026-07-21T16:00:00.000Z', payload: { @@ -91,6 +90,7 @@ describe('terminal protocol', () => { }, }); expect(result.payload).toMatchObject({ request_id: requestId, ok: true, pane_id: '%7' }); + expect(result).not.toHaveProperty('seq'); }); it('accepts a viewer-state reconciliation request after an uncertain focus result', () => { diff --git a/services/control-plane/src/routes/terminal.ts b/services/control-plane/src/routes/terminal.ts index 3329a47b..653c2e87 100644 --- a/services/control-plane/src/routes/terminal.ts +++ b/services/control-plane/src/routes/terminal.ts @@ -107,19 +107,23 @@ export function handleTerminalOutput( } export function handleTerminalNavigationResult( - payload: TerminalNavigationResultMessage['payload'] + payload: TerminalNavigationResultMessage['payload'], + sourceHostId: string ): void { - const pending = pendingNavigationRequests.get(payload.request_id); - if (pending?.channelId === payload.channel_id) { - pendingNavigationRequests.delete(payload.request_id); - recordTerminalNavigation( - pending.operation, - payload.ok ? 'success' : 'failure', - (Date.now() - pending.startedAt) / 1000 - ); - } const channel = activeChannels.get(payload.channel_id); - if (!channel) return; + const pending = pendingNavigationRequests.get(payload.request_id); + if ( + !channel + || channel.hostId !== sourceHostId + || pending?.channelId !== payload.channel_id + ) return; + + pendingNavigationRequests.delete(payload.request_id); + recordTerminalNavigation( + pending.operation, + payload.ok ? 'success' : 'failure', + (Date.now() - pending.startedAt) / 1000 + ); try { const browserMessage: BrowserTerminalNavigationResultMessage = payload.ok @@ -473,7 +477,7 @@ export function registerTerminalRoutes(app: FastifyInstance): void { request_id: correlatedNavigation.request_id, ok: false, message: 'Host agent is not available.', - }); + }, channel.hostId); } resetIdleTimeout(activeChannelId); break; diff --git a/services/control-plane/src/ws/agent.ts b/services/control-plane/src/ws/agent.ts index 80226da7..079138ce 100644 --- a/services/control-plane/src/ws/agent.ts +++ b/services/control-plane/src/ws/agent.ts @@ -252,6 +252,14 @@ async function processAgentMessage( return; } + if (message.type === 'terminal.navigation_result') { + if (!state.hostId) { + throw new Error('Terminal navigation result received before agent authentication'); + } + handleTerminalNavigationResult(message.payload, state.hostId); + return; + } + const seq = message.seq; if (message.type !== 'agent.hello' && seq <= state.lastProcessedSeq) { @@ -323,12 +331,6 @@ async function processAgentMessage( break; } - case 'terminal.navigation_result': { - handleTerminalNavigationResult(message.payload); - sendAck(socket, seq, 'ok'); - break; - } - case 'terminal.attached': case 'terminal.detached': case 'terminal.error': diff --git a/services/control-plane/tests/agentWebSocket.test.ts b/services/control-plane/tests/agentWebSocket.test.ts index 8fbb88aa..b4137247 100644 --- a/services/control-plane/tests/agentWebSocket.test.ts +++ b/services/control-plane/tests/agentWebSocket.test.ts @@ -49,6 +49,7 @@ async function buildServer( subscribeToCommandResults: (send: ReturnType) => () => void; publishTmuxTopology: (authenticatedHostId: string, message: Record) => void; handleTerminalStatus: ReturnType; + handleTerminalNavigationResult: ReturnType; createAuditLog: ReturnType; }> { vi.resetModules(); @@ -104,7 +105,9 @@ async function buildServer( sessionGraph: { upsert: upsertEdge }, })); const handleTerminalStatus = vi.fn(); + const handleTerminalNavigationResult = vi.fn(); vi.doMock('../src/routes/terminal.js', () => ({ + handleTerminalNavigationResult, handleTerminalOutput: vi.fn(), handleTerminalStatus, })); @@ -144,6 +147,7 @@ async function buildServer( subscribeToCommandResults, publishTmuxTopology, handleTerminalStatus, + handleTerminalNavigationResult, createAuditLog, }; } @@ -288,6 +292,41 @@ describe('agent websocket ingest', () => { await app.close(); }); + it('delivers unsequenced terminal navigation results outside the durable cursor', async () => { + const { app, url, handleTerminalNavigationResult } = await buildServer( + vi.fn(async () => undefined) + ); + const socket = new WebSocket(`${url}/v1/agent/connect`, { + headers: { Authorization: 'Bearer test-agent-token' }, + }); + await new Promise((resolve) => socket.once('open', resolve)); + socket.send(JSON.stringify(hello())); + await waitForMessage(socket); + + const payload = { + channel_id: '33333333-3333-4333-8333-333333333333', + request_id: '44444444-4444-4444-8444-444444444444', + ok: true, + pane_id: '%7', + window_index: 2, + zoomed: true, + }; + socket.send(JSON.stringify({ + v: 1, + type: 'terminal.navigation_result', + ts: '2026-07-22T21:35:00Z', + payload, + })); + + await vi.waitFor(() => { + expect(handleTerminalNavigationResult).toHaveBeenCalledWith(payload, hostId); + }); + expect(socket.readyState).toBe(WebSocket.OPEN); + + socket.close(); + await app.close(); + }); + it('keeps the authenticated topology host authoritative over agent payload fields', async () => { const { app, subscribeToTmuxTopology, publishTmuxTopology } = await buildServer( vi.fn(async () => undefined) diff --git a/services/control-plane/tests/terminalRoute.test.ts b/services/control-plane/tests/terminalRoute.test.ts index 3ff0c629..4ff7317f 100644 --- a/services/control-plane/tests/terminalRoute.test.ts +++ b/services/control-plane/tests/terminalRoute.test.ts @@ -143,7 +143,7 @@ async function buildServer(options: { window_index?: number; zoomed?: boolean; message?: string; - }) => void; + }, sourceHostId: string) => void; }> { vi.resetModules(); vi.stubEnv('JWT_SECRET', jwtSecret); @@ -581,6 +581,12 @@ describe('terminal websocket route', () => { }); const response = waitForMessage(socket); + handleTerminalNavigationResult({ + channel_id: String(attach?.payload.channel_id), + request_id: requestId, + ok: false, + message: 'forged cross-host result', + }, '99999999-9999-4999-8999-999999999999'); handleTerminalNavigationResult({ channel_id: String(attach?.payload.channel_id), request_id: requestId, @@ -588,7 +594,7 @@ describe('terminal websocket route', () => { pane_id: '%7', window_index: 2, zoomed: true, - }); + }, hostId); const message = await response; expect(JSON.parse(message.data.toString())).toEqual({ type: 'navigation_result', @@ -624,7 +630,7 @@ describe('terminal websocket route', () => { pane_id: '%7', window_index: 2, zoomed: true, - }); + }, hostId); const stateMessage = await stateResponse; expect(JSON.parse(stateMessage.data.toString())).toEqual({ type: 'navigation_result', diff --git a/tasks/lessons.md b/tasks/lessons.md index 9ed61a50..1c653931 100644 --- a/tasks/lessons.md +++ b/tasks/lessons.md @@ -105,3 +105,7 @@ - Date: 2026-07-22 Correction: The live Focus control became clickable before the terminal viewer had attached, so a valid user click sent no tmux request and was mislabeled as an unconfirmed pane focus; successful focus also waited on topology polling, while channel-scoped acknowledgements unnecessarily entered the durable agent queue. Rule: Gate viewer controls on authoritative attachment state, preserve the real protocol rejection, adopt verified acknowledgement state immediately while topology catches up, and keep browser-channel lifecycle/navigation messages off the durable replay lane. + +- Date: 2026-07-22 + Correction: Marking pane-focus acknowledgements non-durable removed their own disk write but still left them behind the durable sender mutex, so background queue fsync could delay or abandon an otherwise successful tmux focus transaction. + Rule: Browser-channel request/response acknowledgements need an unsequenced live write path that can bypass durable persistence while the WebSocket write itself remains serialized. Prove this with the durable lane held, then measure the exact production acknowledgement rather than inferring success from tmux state alone. diff --git a/tests/fixtures/protocol/terminal-navigation-result.json b/tests/fixtures/protocol/terminal-navigation-result.json index 4aa2ca6e..cf7cfece 100644 --- a/tests/fixtures/protocol/terminal-navigation-result.json +++ b/tests/fixtures/protocol/terminal-navigation-result.json @@ -1 +1 @@ -{"v":1,"type":"terminal.navigation_result","ts":"2026-07-21T16:00:02Z","seq":19,"payload":{"channel_id":"33333333-3333-4333-8333-333333333333","request_id":"44444444-4444-4444-8444-444444444444","ok":true,"pane_id":"%7","window_index":2,"zoomed":true}} +{"v":1,"type":"terminal.navigation_result","ts":"2026-07-21T16:00:02Z","payload":{"channel_id":"33333333-3333-4333-8333-333333333333","request_id":"44444444-4444-4444-8444-444444444444","ok":true,"pane_id":"%7","window_index":2,"zoomed":true}} From d72bd56f3376f68076a6d09daeb0645a510e4336 Mon Sep 17 00:00:00 2001 From: cvsloane Date: Wed, 22 Jul 2026 17:49:45 -0400 Subject: [PATCH 2/2] test(agentd): assert live messages omit sequence --- agents/agentd/internal/ws/client_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/agents/agentd/internal/ws/client_test.go b/agents/agentd/internal/ws/client_test.go index fed5bfaa..6d2b541c 100644 --- a/agents/agentd/internal/ws/client_test.go +++ b/agents/agentd/internal/ws/client_test.go @@ -117,7 +117,7 @@ func TestSendUnsequencedOmitsSequenceField(t *testing.T) { } func TestSendUnsequencedBypassesDurablePersistenceLane(t *testing.T) { - messages := make(chan receivedEnvelope, 1) + messages := make(chan map[string]json.RawMessage, 1) upgrader := websocket.Upgrader{} server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { conn, err := upgrader.Upgrade(w, r, nil) @@ -125,7 +125,7 @@ func TestSendUnsequencedBypassesDurablePersistenceLane(t *testing.T) { return } defer conn.Close() - var message receivedEnvelope + var message map[string]json.RawMessage if err := conn.ReadJSON(&message); err == nil { messages <- message } @@ -150,8 +150,11 @@ func TestSendUnsequencedBypassesDurablePersistenceLane(t *testing.T) { select { case message := <-messages: client.sendMu.Unlock() - if message.Type != "terminal.navigation_result" || message.Seq != 0 { - t.Fatalf("priority message=(%s,%d)", message.Type, message.Seq) + if string(message["type"]) != `"terminal.navigation_result"` { + t.Fatalf("priority message type=%s", message["type"]) + } + if _, exists := message["seq"]; exists { + t.Fatalf("priority message contains seq: %s", message["seq"]) } if err := <-sendDone; err != nil { t.Fatal(err)