From 235d15d451450dbef08411c2e57201e393ead4cb Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Tue, 12 May 2026 13:06:45 +0000 Subject: [PATCH] feat: instrument telemetry for deploy command (CLI + TUI) --- .../commands/deploy/__tests__/utils.test.ts | 54 +++++++++ src/cli/commands/deploy/command.tsx | 114 ++++++++++++------ src/cli/commands/deploy/utils.ts | 33 +++++ .../schemas/__tests__/command-run.test.ts | 6 +- src/cli/telemetry/schemas/command-run.ts | 2 +- src/cli/tui/screens/deploy/useDeployFlow.ts | 20 ++- 6 files changed, 183 insertions(+), 46 deletions(-) create mode 100644 src/cli/commands/deploy/__tests__/utils.test.ts create mode 100644 src/cli/commands/deploy/utils.ts diff --git a/src/cli/commands/deploy/__tests__/utils.test.ts b/src/cli/commands/deploy/__tests__/utils.test.ts new file mode 100644 index 000000000..8bf07311a --- /dev/null +++ b/src/cli/commands/deploy/__tests__/utils.test.ts @@ -0,0 +1,54 @@ +import type { AgentCoreProjectSpec } from '../../../../schema'; +import { computeDeployAttrs } from '../utils.js'; +import { describe, expect, it } from 'vitest'; + +describe('computeDeployAttrs', () => { + it('computes counts from a populated spec', () => { + const projectSpec = { + runtimes: [{}, {}], + memories: [{}], + credentials: [{}, {}, {}], + evaluators: [{}], + onlineEvalConfigs: [{}, {}], + agentCoreGateways: [{ targets: [{}, {}] }, { targets: [{}] }], + policyEngines: [{ policies: [{}, {}] }, { policies: [{}] }], + } as unknown as Partial; + + expect(computeDeployAttrs(projectSpec, 'diff')).toEqual({ + runtime_count: 2, + memory_count: 1, + credential_count: 3, + evaluator_count: 1, + online_eval_count: 2, + gateway_count: 2, + gateway_target_count: 3, + policy_engine_count: 2, + policy_count: 3, + mode: 'diff', + }); + }); + + it('returns zeros for empty spec', () => { + expect(computeDeployAttrs({}, 'deploy')).toEqual({ + runtime_count: 0, + memory_count: 0, + credential_count: 0, + evaluator_count: 0, + online_eval_count: 0, + gateway_count: 0, + gateway_target_count: 0, + policy_engine_count: 0, + policy_count: 0, + mode: 'deploy', + }); + }); + + it('handles dry-run mode', () => { + const projectSpec = { runtimes: [{}] } as unknown as Partial; + const attrs = computeDeployAttrs(projectSpec, 'dry-run'); + + expect(attrs.runtime_count).toBe(1); + expect(attrs.memory_count).toBe(0); + expect(attrs.mode).toBe('dry-run'); + }); +}); diff --git a/src/cli/commands/deploy/command.tsx b/src/cli/commands/deploy/command.tsx index 6251edda8..d735aa4af 100644 --- a/src/cli/commands/deploy/command.tsx +++ b/src/cli/commands/deploy/command.tsx @@ -1,10 +1,12 @@ -import { serializeResult } from '../../../lib'; +import { ConfigIO, serializeResult } from '../../../lib'; import { getErrorMessage } from '../../errors'; +import { withCommandRunTelemetry } from '../../telemetry/cli-command-run.js'; import { COMMAND_DESCRIPTIONS } from '../../tui/copy'; import { requireProject, requireTTY } from '../../tui/guards'; import { DeployScreen } from '../../tui/screens/deploy/DeployScreen'; import { handleDeploy } from './actions'; -import type { DeployOptions } from './types'; +import type { DeployOptions, DeployResult } from './types'; +import { DEFAULT_DEPLOY_ATTRS, computeDeployAttrs } from './utils'; import { validateDeployOptions } from './validate'; import type { Command } from '@commander-js/extra-typings'; import { Text, render } from 'ink'; @@ -39,6 +41,45 @@ async function handleDeployCLI(options: DeployOptions): Promise { process.exit(1); } + // Compute attrs upfront from project spec (available before deploy) + const mode = options.diff ? 'diff' : options.plan ? 'dry-run' : 'deploy'; + const attrs = await new ConfigIO() + .readProjectSpec() + .then(spec => computeDeployAttrs(spec, mode)) + .catch(() => ({ ...DEFAULT_DEPLOY_ATTRS, mode }) as const); + + const { deployResult } = await withCommandRunTelemetry('deploy', attrs, async () => { + const result = await executeDeploy(options).catch( + (e): DeployResult => ({ success: false, error: e instanceof Error ? e : new Error(getErrorMessage(e)) }) + ); + if (!result.success) { + return { success: false as const, error: result.error, deployResult: result }; + } + return { success: true as const, deployResult: result }; + }); + + // ALL output happens here, after telemetry + if (!deployResult.success) { + if (options.json) { + console.log(JSON.stringify(serializeResult(deployResult))); + } else { + console.error(deployResult.error.message); + if (deployResult.logPath) { + console.error(`Log: ${deployResult.logPath}`); + } + } + process.exit(1); + } + + printDeployResult(deployResult, options); + + if (deployResult.postDeployWarnings && deployResult.postDeployWarnings.length > 0) { + process.exit(2); + } + process.exit(0); +} + +async function executeDeploy(options: DeployOptions): Promise { let spinner: NodeJS.Timeout | undefined; // Progress callback for --progress mode @@ -85,55 +126,52 @@ async function handleDeployCLI(options: DeployOptions): Promise { process.stdout.write('\r\x1b[K'); } + return result; +} + +function printDeployResult(result: DeployResult & { success: true }, options: DeployOptions): void { if (options.json) { - console.log(JSON.stringify(serializeResult(result))); - } else if (result.success) { - if (options.diff) { - console.log(`\n✓ Diff complete for '${result.targetName}' (stack: ${result.stackName})`); - } else if (options.plan) { - console.log(`\n✓ Dry run complete for '${result.targetName}' (stack: ${result.stackName})`); - console.log('\nRun `agentcore deploy` to deploy.'); - } else { - console.log(`\n✓ Deployed to '${result.targetName}' (stack: ${result.stackName})`); + console.log(JSON.stringify(result)); + return; + } - // Show stack outputs in non-JSON mode - if (result.outputs && Object.keys(result.outputs).length > 0) { - console.log('\nOutputs:'); - for (const [key, value] of Object.entries(result.outputs)) { - console.log(` ${key}: ${value}`); - } - } + if (options.diff) { + console.log(`\n✓ Diff complete for '${result.targetName}' (stack: ${result.stackName})`); + } else if (options.plan) { + console.log(`\n✓ Dry run complete for '${result.targetName}' (stack: ${result.stackName})`); + console.log('\nRun `agentcore deploy` to deploy.'); + } else { + console.log(`\n✓ Deployed to '${result.targetName}' (stack: ${result.stackName})`); - if (result.postDeployWarnings && result.postDeployWarnings.length > 0) { - console.log('\n⚠ Post-deploy warnings:'); - for (const warning of result.postDeployWarnings) { - console.log(` ${warning}`); - } + // Show stack outputs in non-JSON mode + if (result.outputs && Object.keys(result.outputs).length > 0) { + console.log('\nOutputs:'); + for (const [key, value] of Object.entries(result.outputs)) { + console.log(` ${key}: ${value}`); } + } - if (result.notes && result.notes.length > 0) { - for (const note of result.notes) { - console.log(`\nNote: ${note}`); - } + if (result.postDeployWarnings && result.postDeployWarnings.length > 0) { + console.log('\n⚠ Post-deploy warnings:'); + for (const warning of result.postDeployWarnings) { + console.log(` ${warning}`); } + } - if (result.nextSteps && result.nextSteps.length > 0) { - console.log(`Next: ${result.nextSteps.join(' | ')}`); + if (result.notes && result.notes.length > 0) { + for (const note of result.notes) { + console.log(`\nNote: ${note}`); } } - if (result.logPath) { - console.log(`\nLog: ${result.logPath}`); - } - } else { - console.error(result.error.message); - if (result.logPath) { - console.error(`Log: ${result.logPath}`); + if (result.nextSteps && result.nextSteps.length > 0) { + console.log(`Next: ${result.nextSteps.join(' | ')}`); } } - const hasPostDeployWarnings = result.success && result.postDeployWarnings && result.postDeployWarnings.length > 0; - process.exit(result.success ? (hasPostDeployWarnings ? 2 : 0) : 1); + if (result.logPath) { + console.log(`\nLog: ${result.logPath}`); + } } export const registerDeploy = (program: Command) => { diff --git a/src/cli/commands/deploy/utils.ts b/src/cli/commands/deploy/utils.ts new file mode 100644 index 000000000..d0db1ec08 --- /dev/null +++ b/src/cli/commands/deploy/utils.ts @@ -0,0 +1,33 @@ +import type { AgentCoreProjectSpec } from '../../../schema'; + +export type DeployMode = 'deploy' | 'dry-run' | 'diff'; + +export const DEFAULT_DEPLOY_ATTRS = { + runtime_count: 0, + memory_count: 0, + credential_count: 0, + evaluator_count: 0, + online_eval_count: 0, + gateway_count: 0, + gateway_target_count: 0, + policy_engine_count: 0, + policy_count: 0, + mode: 'deploy' as DeployMode, +}; + +export function computeDeployAttrs(projectSpec: Partial, mode: DeployMode) { + const gateways = projectSpec.agentCoreGateways ?? []; + const policyEngines = projectSpec.policyEngines ?? []; + return { + runtime_count: (projectSpec.runtimes ?? []).length, + memory_count: (projectSpec.memories ?? []).length, + credential_count: (projectSpec.credentials ?? []).length, + evaluator_count: (projectSpec.evaluators ?? []).length, + online_eval_count: (projectSpec.onlineEvalConfigs ?? []).length, + gateway_count: gateways.length, + gateway_target_count: gateways.reduce((sum, g) => sum + (g.targets ?? []).length, 0), + policy_engine_count: policyEngines.length, + policy_count: policyEngines.reduce((sum, pe) => sum + (pe.policies ?? []).length, 0), + mode, + }; +} diff --git a/src/cli/telemetry/schemas/__tests__/command-run.test.ts b/src/cli/telemetry/schemas/__tests__/command-run.test.ts index 110df1284..2df12d271 100644 --- a/src/cli/telemetry/schemas/__tests__/command-run.test.ts +++ b/src/cli/telemetry/schemas/__tests__/command-run.test.ts @@ -47,7 +47,7 @@ describe('COMMAND_SCHEMAS', () => { gateway_target_count: 3, policy_engine_count: 0, policy_count: 0, - has_diff: true, + mode: 'diff', }; expect(COMMAND_SCHEMAS.deploy.parse(attrs)).toEqual(attrs); }); @@ -64,7 +64,7 @@ describe('COMMAND_SCHEMAS', () => { gateway_target_count: 0, policy_engine_count: 0, policy_count: 0, - has_diff: false, + mode: 'deploy', }) ).toThrow(); }); @@ -81,7 +81,7 @@ describe('COMMAND_SCHEMAS', () => { gateway_target_count: 0, policy_engine_count: 0, policy_count: 0, - has_diff: false, + mode: 'deploy', }) ).toThrow(); }); diff --git a/src/cli/telemetry/schemas/command-run.ts b/src/cli/telemetry/schemas/command-run.ts index 6386e297d..bf79f42eb 100644 --- a/src/cli/telemetry/schemas/command-run.ts +++ b/src/cli/telemetry/schemas/command-run.ts @@ -100,7 +100,7 @@ const DeployAttrs = safeSchema({ gateway_target_count: Count, policy_engine_count: Count, policy_count: Count, - has_diff: z.boolean(), + mode: z.enum(['deploy', 'dry-run', 'diff']), }); const DevAttrs = safeSchema({ diff --git a/src/cli/tui/screens/deploy/useDeployFlow.ts b/src/cli/tui/screens/deploy/useDeployFlow.ts index fa1cf3906..637a5d832 100644 --- a/src/cli/tui/screens/deploy/useDeployFlow.ts +++ b/src/cli/tui/screens/deploy/useDeployFlow.ts @@ -11,6 +11,7 @@ import { parsePolicyEngineOutputs, parsePolicyOutputs, } from '../../../cloudformation'; +import { DEFAULT_DEPLOY_ATTRS, computeDeployAttrs } from '../../../commands/deploy/utils.js'; import { getErrorMessage, isChangesetInProgressError, isExpiredTokenError } from '../../../errors'; import { ExecLogger } from '../../../logging'; import { performStackTeardown, setupTransactionSearch } from '../../../operations/deploy'; @@ -22,6 +23,7 @@ import { } from '../../../operations/deploy/post-deploy-config-bundles'; import { setupHttpGateways } from '../../../operations/deploy/post-deploy-http-gateways'; import { enableOnlineEvalConfigs } from '../../../operations/deploy/post-deploy-online-evals'; +import { withCommandRunTelemetry } from '../../../telemetry/cli-command-run.js'; import { type StackDiffSummary, type Step, @@ -550,7 +552,9 @@ export function useDeployFlow(options: DeployFlowOptions = {}): DeployFlowState if (deployStep.status !== 'pending') return; if (!cdkToolkitWrapper) return; - const run = async () => { + const attrs = context ? computeDeployAttrs(context.projectSpec, 'deploy') : { ...DEFAULT_DEPLOY_ATTRS }; + + const run = async (): Promise<{ success: true } | { success: false; error: Error }> => { // Run diff before deploy to capture pre-deploy differences if (!isDiffRunningRef.current) { isDiffRunningRef.current = true; @@ -669,6 +673,7 @@ export function useDeployFlow(options: DeployFlowOptions = {}): DeployFlowState // Mark both steps as success (in case CFn events were never received) setPublishAssetsStep(prev => ({ ...prev, status: 'success' })); setDeployStep(prev => ({ ...prev, status: 'success' })); + return { success: true } as const; } catch (err) { const errorMsg = getErrorMessage(err); @@ -700,6 +705,7 @@ export function useDeployFlow(options: DeployFlowOptions = {}): DeployFlowState error: logger.getFailureMessage('Publish assets'), })); } + return { success: false, error: err instanceof Error ? err : new Error(errorMsg) } as const; } finally { // Disable verbose output and clear callback after deploy switchableIoHost?.setVerbose(false); @@ -709,7 +715,7 @@ export function useDeployFlow(options: DeployFlowOptions = {}): DeployFlowState } }; - void run(); + void withCommandRunTelemetry('deploy', attrs, run); }, [ preflight.phase, cdkToolkitWrapper, @@ -734,7 +740,11 @@ export function useDeployFlow(options: DeployFlowOptions = {}): DeployFlowState if (diffStep.status !== 'pending') return; if (!cdkToolkitWrapper) return; - const run = async () => { + const attrs = context + ? computeDeployAttrs(context.projectSpec, 'diff') + : { ...DEFAULT_DEPLOY_ATTRS, mode: 'diff' as const }; + + const run = async (): Promise<{ success: true } | { success: false; error: Error }> => { setDiffStep(prev => ({ ...prev, status: 'running' })); setShouldStartDeploy(false); setDiffSummaries([]); @@ -755,6 +765,7 @@ export function useDeployFlow(options: DeployFlowOptions = {}): DeployFlowState logger.endStep('success'); logger.finalize(true); setDiffStep(prev => ({ ...prev, status: 'success' })); + return { success: true }; } catch (err) { const errorMsg = getErrorMessage(err); logger.endStep('error', errorMsg); @@ -769,6 +780,7 @@ export function useDeployFlow(options: DeployFlowOptions = {}): DeployFlowState status: 'error', error: logger.getFailureMessage('Run CDK diff'), })); + return { success: false, error: err instanceof Error ? err : new Error(errorMsg) }; } finally { switchableIoHost?.setVerbose(false); switchableIoHost?.setOnRawMessage(null); @@ -776,7 +788,7 @@ export function useDeployFlow(options: DeployFlowOptions = {}): DeployFlowState } }; - void run(); + void withCommandRunTelemetry('deploy', attrs, run); }, [ diffMode, preflight.phase,