diff --git a/docs/commands.md b/docs/commands.md index f6f15b9ae..aa50bd6ff 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -502,17 +502,19 @@ agentcore dev list-tools agentcore dev call-tool --tool myTool --input '{"arg": "value"}' ``` -| Flag / Argument | Description | -| ---------------------- | --------------------------------------------------------------------- | -| `[prompt]` | Send a prompt to a running dev server | -| `-p, --port ` | Port (default: 8080; MCP uses 8000, A2A uses 9000) | -| `-r, --runtime ` | Runtime to run or invoke (required if multiple runtimes) | -| `-s, --stream` | Stream response when invoking | -| `-l, --logs` | Non-interactive stdout logging | -| `--tool ` | MCP tool name (with `call-tool` prompt) | -| `--input ` | MCP tool arguments as JSON (with `--tool`) | -| `-H, --header ` | Custom header (`"Name: Value"`, repeatable) | -| `--exec` | Execute a shell command in the running dev container (Container only) | +| Flag / Argument | Description | +| ----------------------- | ---------------------------------------------------------------------------------- | +| `[prompt]` | Send a prompt to a running dev server | +| `-p, --port ` | Port (default: 8080; MCP uses 8000, A2A uses 9000) | +| `-r, --runtime ` | Runtime to run or invoke (required if multiple runtimes) | +| `-s, --stream` | Stream response when invoking | +| `-l, --logs` | Non-interactive stdout logging | +| `--tool ` | MCP tool name (with `call-tool` prompt) | +| `--input ` | MCP tool arguments as JSON (with `--tool`) | +| `-H, --header ` | Custom header (`"Name: Value"`, repeatable) | +| `--exec` | Execute a shell command in the running dev container (Container only) | +| `--no-traces` | Disable local OTEL trace collection entirely | +| `--otel-endpoint ` | Forward agent traces to a custom OTLP/HTTP endpoint instead of the local collector | ### invoke diff --git a/docs/local-development.md b/docs/local-development.md index d13033768..5ac0772e0 100644 --- a/docs/local-development.md +++ b/docs/local-development.md @@ -109,6 +109,53 @@ See [Container Builds](container-builds.md) for full details on container develo Memory requires deployment to test fully. For local testing, you can mock these dependencies in your agent code. +## Telemetry + +In dev mode the CLI automatically starts a local OTLP/HTTP collector and injects the standard OpenTelemetry environment +variables into the agent process. Traces are persisted to `agentcore/.cli/traces/` and displayed in the web UI. + +### Custom OTLP Endpoint + +To forward traces to your own backend (Jaeger, Grafana Tempo, a cloud collector, etc.) instead of the local collector: + +```bash +agentcore dev --otel-endpoint http://localhost:4318 +``` + +When `--otel-endpoint` is set: + +- The local in-process collector is **not** started — no port is bound by the CLI +- `OTEL_EXPORTER_OTLP_ENDPOINT` is set to your URL +- All other OTEL env vars are injected as normal +- Traces will **not** appear in the web UI traces panel (they go directly to your backend) + +The endpoint at startup is printed to stdout: + +``` +OTEL traces → http://localhost:4318 +``` + +### Disabling Traces + +To disable telemetry collection entirely: + +```bash +agentcore dev --no-traces +``` + +No OTEL env vars are injected into the agent process. + +### Trace Storage + +When using the default local collector, traces are persisted as OTLP JSON Lines files: + +``` +agentcore/.cli/traces/otlp/ +└── -.otlp.jsonl +``` + +Traces survive dev server restarts and are available in the web UI traces panel across sessions. + ## Gateway Environment Variables When you have deployed gateways, `agentcore dev` automatically injects gateway environment variables into your local diff --git a/src/cli/commands/dev/__tests__/dev.test.ts b/src/cli/commands/dev/__tests__/dev.test.ts index 3947f6784..392121d6f 100644 --- a/src/cli/commands/dev/__tests__/dev.test.ts +++ b/src/cli/commands/dev/__tests__/dev.test.ts @@ -82,4 +82,23 @@ describe('dev command', () => { expect(result.stdout.includes('--stream'), 'Should show --stream option').toBeTruthy(); }); }); + + describe('--otel-endpoint', () => { + it('is documented in help', async () => { + const result = await runCLI(['dev', '--help'], process.cwd()); + + expect(result.exitCode).toBe(0); + expect(result.stdout.includes('--otel-endpoint'), 'Should show --otel-endpoint option').toBeTruthy(); + }); + + it('help text mentions custom OTLP endpoint', async () => { + const result = await runCLI(['dev', '--help'], process.cwd()); + + expect(result.exitCode).toBe(0); + expect( + result.stdout.toLowerCase().includes('otlp') || result.stdout.toLowerCase().includes('endpoint'), + 'Should describe the OTLP endpoint purpose' + ).toBeTruthy(); + }); + }); }); diff --git a/src/cli/commands/dev/browser-mode.ts b/src/cli/commands/dev/browser-mode.ts index 3846dea85..ed62c1fed 100644 --- a/src/cli/commands/dev/browser-mode.ts +++ b/src/cli/commands/dev/browser-mode.ts @@ -98,13 +98,15 @@ export interface BrowserModeOptions { otelEnvVars?: Record; /** OTEL collector instance for local trace collection */ collector?: OtelCollector; + /** Custom OTLP endpoint to forward traces to instead of the local collector */ + otelEndpoint?: string; } /** * Standalone entry point for launching browser dev mode from the TUI. * Handles all setup (project loading, OTEL collector, etc.) internally. */ -export async function launchBrowserDev(): Promise { +export async function launchBrowserDev(otelEndpoint?: string): Promise { const workingDir = getWorkingDirectory(); const project = await loadProjectConfig(workingDir); @@ -115,7 +117,7 @@ export async function launchBrowserDev(): Promise { const configRoot = findConfigRoot(workingDir); const persistTracesDir = path.join(configRoot ?? workingDir, '.cli', 'traces'); - const { collector, otelEnvVars } = await startOtelCollector(persistTracesDir); + const { collector, otelEnvVars } = await startOtelCollector(persistTracesDir, otelEndpoint); await runBrowserMode({ workingDir, @@ -123,6 +125,7 @@ export async function launchBrowserDev(): Promise { port: 8080, otelEnvVars, collector, + otelEndpoint, }); } diff --git a/src/cli/commands/dev/command.tsx b/src/cli/commands/dev/command.tsx index eac485d8c..9cb233037 100644 --- a/src/cli/commands/dev/command.tsx +++ b/src/cli/commands/dev/command.tsx @@ -186,6 +186,10 @@ export const registerDev = (program: Command) => { ) .option('-b, --no-browser', 'Use terminal TUI instead of web-based chat UI') .option('--no-traces', 'Disable local OTEL trace collection') + .option( + '--otel-endpoint ', + 'Forward agent traces to a custom OTLP/HTTP endpoint instead of the local collector (e.g. http://localhost:4318)' + ) .action(async (positionalPrompt: string | undefined, opts) => { try { @@ -297,9 +301,12 @@ export const registerDev = (program: Command) => { if (opts.traces !== false) { const persistTracesDir = path.join(configRoot ?? workingDir, '.cli', 'traces'); - const otelResult = await startOtelCollector(persistTracesDir); + const otelResult = await startOtelCollector(persistTracesDir, opts.otelEndpoint); collector = otelResult.collector; otelEnvVars = otelResult.otelEnvVars; + if (opts.otelEndpoint) { + console.log(`OTEL traces → ${opts.otelEndpoint}`); + } } // If --logs provided, run non-interactive mode @@ -422,6 +429,7 @@ export const registerDev = (program: Command) => { agentName: opts.runtime, otelEnvVars, collector, + otelEndpoint: opts.otelEndpoint, }); } catch (error) { render(Error: {getErrorMessage(error)}); diff --git a/src/cli/operations/dev/otel/__tests__/collector.test.ts b/src/cli/operations/dev/otel/__tests__/collector.test.ts new file mode 100644 index 000000000..09184aa01 --- /dev/null +++ b/src/cli/operations/dev/otel/__tests__/collector.test.ts @@ -0,0 +1,117 @@ +import { startOtelCollector } from '../collector.js'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +// --------------------------------------------------------------------------- +// Mock node:http createServer so tests don't bind real sockets. +// The mock server immediately calls the listen callback (port available). +// --------------------------------------------------------------------------- + +const { mockListen, mockClose, mockOn } = vi.hoisted(() => { + const mockListen = vi.fn((_port: number, _host: string, cb: () => void) => { + queueMicrotask(cb); + }); + const mockClose = vi.fn(); + const mockOn = vi.fn(); + return { mockListen, mockClose, mockOn }; +}); + +vi.mock('node:http', async importOriginal => { + const actual = await importOriginal(); + return { + ...actual, + createServer: vi.fn(() => ({ + listen: mockListen, + close: mockClose, + on: mockOn, + })), + }; +}); + +afterEach(() => { + vi.clearAllMocks(); +}); + +// --------------------------------------------------------------------------- +// startOtelCollector — without custom endpoint (default behaviour) +// --------------------------------------------------------------------------- + +describe('startOtelCollector', () => { + describe('without custom endpoint (default behaviour)', () => { + it('returns a defined collector instance', async () => { + const { collector } = await startOtelCollector('/tmp/traces'); + + expect(collector).toBeDefined(); + }); + + it('sets OTEL_EXPORTER_OTLP_ENDPOINT to a local 127.0.0.1 address', async () => { + const { otelEnvVars } = await startOtelCollector('/tmp/traces'); + + expect(otelEnvVars.OTEL_EXPORTER_OTLP_ENDPOINT).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/); + }); + + it('sets all required OTEL env vars', async () => { + const { otelEnvVars } = await startOtelCollector('/tmp/traces'); + + expect(otelEnvVars.OTEL_EXPORTER_OTLP_PROTOCOL).toBe('http/protobuf'); + expect(otelEnvVars.OTEL_METRICS_EXPORTER).toBe('none'); + expect(otelEnvVars.AGENT_OBSERVABILITY_ENABLED).toBe('true'); + expect(otelEnvVars.OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT).toBe('true'); + expect(otelEnvVars.OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED).toBe('true'); + }); + + it('starts the HTTP server (binds a port)', async () => { + await startOtelCollector('/tmp/traces'); + + expect(mockListen).toHaveBeenCalledOnce(); + }); + }); + + // --------------------------------------------------------------------------- + // startOtelCollector — with custom endpoint + // --------------------------------------------------------------------------- + + describe('with custom endpoint', () => { + const CUSTOM_ENDPOINT = 'http://localhost:5388/otel/default'; + + it('returns collector as undefined', async () => { + const { collector } = await startOtelCollector('/tmp/traces', CUSTOM_ENDPOINT); + + expect(collector).toBeUndefined(); + }); + + it('does NOT start a local HTTP server', async () => { + await startOtelCollector('/tmp/traces', CUSTOM_ENDPOINT); + + expect(mockListen).not.toHaveBeenCalled(); + }); + + it('sets OTEL_EXPORTER_OTLP_ENDPOINT to the custom endpoint', async () => { + const { otelEnvVars } = await startOtelCollector('/tmp/traces', CUSTOM_ENDPOINT); + + expect(otelEnvVars.OTEL_EXPORTER_OTLP_ENDPOINT).toBe(CUSTOM_ENDPOINT); + }); + + it('does not set a local 127.0.0.1 address as the endpoint', async () => { + const { otelEnvVars } = await startOtelCollector('/tmp/traces', CUSTOM_ENDPOINT); + + expect(otelEnvVars.OTEL_EXPORTER_OTLP_ENDPOINT).not.toMatch(/127\.0\.0\.1/); + }); + + it('still sets all other required OTEL env vars', async () => { + const { otelEnvVars } = await startOtelCollector('/tmp/traces', CUSTOM_ENDPOINT); + + expect(otelEnvVars.OTEL_EXPORTER_OTLP_PROTOCOL).toBe('http/protobuf'); + expect(otelEnvVars.OTEL_METRICS_EXPORTER).toBe('none'); + expect(otelEnvVars.AGENT_OBSERVABILITY_ENABLED).toBe('true'); + expect(otelEnvVars.OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT).toBe('true'); + expect(otelEnvVars.OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED).toBe('true'); + }); + + it('works with a trailing-path endpoint URL', async () => { + const endpoint = 'http://my-collector.internal:4317/v1/traces'; + const { otelEnvVars } = await startOtelCollector('/tmp/traces', endpoint); + + expect(otelEnvVars.OTEL_EXPORTER_OTLP_ENDPOINT).toBe(endpoint); + }); + }); +}); diff --git a/src/cli/operations/dev/otel/collector.ts b/src/cli/operations/dev/otel/collector.ts index ee3ad7838..2661032c1 100644 --- a/src/cli/operations/dev/otel/collector.ts +++ b/src/cli/operations/dev/otel/collector.ts @@ -288,11 +288,31 @@ function readBodyAsBuffer(req: IncomingMessage): Promise { /** * Start an OTEL collector and return it along with the env vars agents need * to export traces to it. + * + * When `customEndpoint` is provided the local in-process collector is NOT + * started. Instead the env vars are set to point at the custom endpoint and + * `collector` is returned as `undefined`. Traces will not be available in the + * web UI in this mode — they are forwarded directly to the external backend. */ -export async function startOtelCollector(persistTracesDir: string): Promise<{ - collector: OtelCollector; +export async function startOtelCollector( + persistTracesDir: string, + customEndpoint?: string +): Promise<{ + collector: OtelCollector | undefined; otelEnvVars: Record; }> { + if (customEndpoint) { + const otelEnvVars: Record = { + OTEL_EXPORTER_OTLP_ENDPOINT: customEndpoint, + OTEL_EXPORTER_OTLP_PROTOCOL: 'http/protobuf', + OTEL_METRICS_EXPORTER: 'none', + AGENT_OBSERVABILITY_ENABLED: 'true', + OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: 'true', + OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED: 'true', + }; + return { collector: undefined, otelEnvVars }; + } + const collector = new OtelCollector({ persistTracesDir }); const collectorPort = await collector.start();