From 630764f311c8911176f9daf4153c2c243c398d16 Mon Sep 17 00:00:00 2001 From: Massimiliano Angelino Date: Wed, 29 Apr 2026 08:29:01 +0100 Subject: [PATCH 1/3] feat(dev): add --otel-endpoint flag to forward traces to custom OTLP backend Add a new --otel-endpoint option to agentcore dev that lets users send agent traces to an external OTLP/HTTP collector (e.g. Jaeger, Grafana Tempo) instead of the built-in local collector. When --otel-endpoint is provided: - The local in-process OtelCollector is not started (no port bound) - OTEL_EXPORTER_OTLP_ENDPOINT is set to the supplied URL - All other OTEL env vars are injected as normal - collector is returned as undefined (traces panel in web UI will be empty) The existing --no-traces flag continues to disable telemetry entirely. Changes: - src/cli/operations/dev/otel/collector.ts: startOtelCollector accepts optional customEndpoint; skips local server when provided - src/cli/commands/dev/command.tsx: add --otel-endpoint flag, log the endpoint at startup, pass through to runBrowserMode - src/cli/commands/dev/browser-mode.ts: thread otelEndpoint through BrowserModeOptions and launchBrowserDev - src/cli/commands/dev/__tests__/dev.test.ts: verify flag appears in help - src/cli/operations/dev/otel/__tests__/collector.test.ts: 10 unit tests covering both default and custom-endpoint behaviour --- src/cli/commands/dev/__tests__/dev.test.ts | 19 +++ src/cli/commands/dev/browser-mode.ts | 7 +- src/cli/commands/dev/command.tsx | 10 +- .../dev/otel/__tests__/collector.test.ts | 117 ++++++++++++++++++ src/cli/operations/dev/otel/collector.ts | 24 +++- 5 files changed, 172 insertions(+), 5 deletions(-) create mode 100644 src/cli/operations/dev/otel/__tests__/collector.test.ts diff --git a/src/cli/commands/dev/__tests__/dev.test.ts b/src/cli/commands/dev/__tests__/dev.test.ts index 3947f6784..392121d6f 100644 --- a/src/cli/commands/dev/__tests__/dev.test.ts +++ b/src/cli/commands/dev/__tests__/dev.test.ts @@ -82,4 +82,23 @@ describe('dev command', () => { expect(result.stdout.includes('--stream'), 'Should show --stream option').toBeTruthy(); }); }); + + describe('--otel-endpoint', () => { + it('is documented in help', async () => { + const result = await runCLI(['dev', '--help'], process.cwd()); + + expect(result.exitCode).toBe(0); + expect(result.stdout.includes('--otel-endpoint'), 'Should show --otel-endpoint option').toBeTruthy(); + }); + + it('help text mentions custom OTLP endpoint', async () => { + const result = await runCLI(['dev', '--help'], process.cwd()); + + expect(result.exitCode).toBe(0); + expect( + result.stdout.toLowerCase().includes('otlp') || result.stdout.toLowerCase().includes('endpoint'), + 'Should describe the OTLP endpoint purpose' + ).toBeTruthy(); + }); + }); }); diff --git a/src/cli/commands/dev/browser-mode.ts b/src/cli/commands/dev/browser-mode.ts index 3846dea85..ed62c1fed 100644 --- a/src/cli/commands/dev/browser-mode.ts +++ b/src/cli/commands/dev/browser-mode.ts @@ -98,13 +98,15 @@ export interface BrowserModeOptions { otelEnvVars?: Record; /** OTEL collector instance for local trace collection */ collector?: OtelCollector; + /** Custom OTLP endpoint to forward traces to instead of the local collector */ + otelEndpoint?: string; } /** * Standalone entry point for launching browser dev mode from the TUI. * Handles all setup (project loading, OTEL collector, etc.) internally. */ -export async function launchBrowserDev(): Promise { +export async function launchBrowserDev(otelEndpoint?: string): Promise { const workingDir = getWorkingDirectory(); const project = await loadProjectConfig(workingDir); @@ -115,7 +117,7 @@ export async function launchBrowserDev(): Promise { const configRoot = findConfigRoot(workingDir); const persistTracesDir = path.join(configRoot ?? workingDir, '.cli', 'traces'); - const { collector, otelEnvVars } = await startOtelCollector(persistTracesDir); + const { collector, otelEnvVars } = await startOtelCollector(persistTracesDir, otelEndpoint); await runBrowserMode({ workingDir, @@ -123,6 +125,7 @@ export async function launchBrowserDev(): Promise { port: 8080, otelEnvVars, collector, + otelEndpoint, }); } diff --git a/src/cli/commands/dev/command.tsx b/src/cli/commands/dev/command.tsx index eac485d8c..9cb233037 100644 --- a/src/cli/commands/dev/command.tsx +++ b/src/cli/commands/dev/command.tsx @@ -186,6 +186,10 @@ export const registerDev = (program: Command) => { ) .option('-b, --no-browser', 'Use terminal TUI instead of web-based chat UI') .option('--no-traces', 'Disable local OTEL trace collection') + .option( + '--otel-endpoint ', + 'Forward agent traces to a custom OTLP/HTTP endpoint instead of the local collector (e.g. http://localhost:4318)' + ) .action(async (positionalPrompt: string | undefined, opts) => { try { @@ -297,9 +301,12 @@ export const registerDev = (program: Command) => { if (opts.traces !== false) { const persistTracesDir = path.join(configRoot ?? workingDir, '.cli', 'traces'); - const otelResult = await startOtelCollector(persistTracesDir); + const otelResult = await startOtelCollector(persistTracesDir, opts.otelEndpoint); collector = otelResult.collector; otelEnvVars = otelResult.otelEnvVars; + if (opts.otelEndpoint) { + console.log(`OTEL traces → ${opts.otelEndpoint}`); + } } // If --logs provided, run non-interactive mode @@ -422,6 +429,7 @@ export const registerDev = (program: Command) => { agentName: opts.runtime, otelEnvVars, collector, + otelEndpoint: opts.otelEndpoint, }); } catch (error) { render(Error: {getErrorMessage(error)}); diff --git a/src/cli/operations/dev/otel/__tests__/collector.test.ts b/src/cli/operations/dev/otel/__tests__/collector.test.ts new file mode 100644 index 000000000..09184aa01 --- /dev/null +++ b/src/cli/operations/dev/otel/__tests__/collector.test.ts @@ -0,0 +1,117 @@ +import { startOtelCollector } from '../collector.js'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +// --------------------------------------------------------------------------- +// Mock node:http createServer so tests don't bind real sockets. +// The mock server immediately calls the listen callback (port available). +// --------------------------------------------------------------------------- + +const { mockListen, mockClose, mockOn } = vi.hoisted(() => { + const mockListen = vi.fn((_port: number, _host: string, cb: () => void) => { + queueMicrotask(cb); + }); + const mockClose = vi.fn(); + const mockOn = vi.fn(); + return { mockListen, mockClose, mockOn }; +}); + +vi.mock('node:http', async importOriginal => { + const actual = await importOriginal(); + return { + ...actual, + createServer: vi.fn(() => ({ + listen: mockListen, + close: mockClose, + on: mockOn, + })), + }; +}); + +afterEach(() => { + vi.clearAllMocks(); +}); + +// --------------------------------------------------------------------------- +// startOtelCollector — without custom endpoint (default behaviour) +// --------------------------------------------------------------------------- + +describe('startOtelCollector', () => { + describe('without custom endpoint (default behaviour)', () => { + it('returns a defined collector instance', async () => { + const { collector } = await startOtelCollector('/tmp/traces'); + + expect(collector).toBeDefined(); + }); + + it('sets OTEL_EXPORTER_OTLP_ENDPOINT to a local 127.0.0.1 address', async () => { + const { otelEnvVars } = await startOtelCollector('/tmp/traces'); + + expect(otelEnvVars.OTEL_EXPORTER_OTLP_ENDPOINT).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/); + }); + + it('sets all required OTEL env vars', async () => { + const { otelEnvVars } = await startOtelCollector('/tmp/traces'); + + expect(otelEnvVars.OTEL_EXPORTER_OTLP_PROTOCOL).toBe('http/protobuf'); + expect(otelEnvVars.OTEL_METRICS_EXPORTER).toBe('none'); + expect(otelEnvVars.AGENT_OBSERVABILITY_ENABLED).toBe('true'); + expect(otelEnvVars.OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT).toBe('true'); + expect(otelEnvVars.OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED).toBe('true'); + }); + + it('starts the HTTP server (binds a port)', async () => { + await startOtelCollector('/tmp/traces'); + + expect(mockListen).toHaveBeenCalledOnce(); + }); + }); + + // --------------------------------------------------------------------------- + // startOtelCollector — with custom endpoint + // --------------------------------------------------------------------------- + + describe('with custom endpoint', () => { + const CUSTOM_ENDPOINT = 'http://localhost:5388/otel/default'; + + it('returns collector as undefined', async () => { + const { collector } = await startOtelCollector('/tmp/traces', CUSTOM_ENDPOINT); + + expect(collector).toBeUndefined(); + }); + + it('does NOT start a local HTTP server', async () => { + await startOtelCollector('/tmp/traces', CUSTOM_ENDPOINT); + + expect(mockListen).not.toHaveBeenCalled(); + }); + + it('sets OTEL_EXPORTER_OTLP_ENDPOINT to the custom endpoint', async () => { + const { otelEnvVars } = await startOtelCollector('/tmp/traces', CUSTOM_ENDPOINT); + + expect(otelEnvVars.OTEL_EXPORTER_OTLP_ENDPOINT).toBe(CUSTOM_ENDPOINT); + }); + + it('does not set a local 127.0.0.1 address as the endpoint', async () => { + const { otelEnvVars } = await startOtelCollector('/tmp/traces', CUSTOM_ENDPOINT); + + expect(otelEnvVars.OTEL_EXPORTER_OTLP_ENDPOINT).not.toMatch(/127\.0\.0\.1/); + }); + + it('still sets all other required OTEL env vars', async () => { + const { otelEnvVars } = await startOtelCollector('/tmp/traces', CUSTOM_ENDPOINT); + + expect(otelEnvVars.OTEL_EXPORTER_OTLP_PROTOCOL).toBe('http/protobuf'); + expect(otelEnvVars.OTEL_METRICS_EXPORTER).toBe('none'); + expect(otelEnvVars.AGENT_OBSERVABILITY_ENABLED).toBe('true'); + expect(otelEnvVars.OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT).toBe('true'); + expect(otelEnvVars.OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED).toBe('true'); + }); + + it('works with a trailing-path endpoint URL', async () => { + const endpoint = 'http://my-collector.internal:4317/v1/traces'; + const { otelEnvVars } = await startOtelCollector('/tmp/traces', endpoint); + + expect(otelEnvVars.OTEL_EXPORTER_OTLP_ENDPOINT).toBe(endpoint); + }); + }); +}); diff --git a/src/cli/operations/dev/otel/collector.ts b/src/cli/operations/dev/otel/collector.ts index ee3ad7838..2661032c1 100644 --- a/src/cli/operations/dev/otel/collector.ts +++ b/src/cli/operations/dev/otel/collector.ts @@ -288,11 +288,31 @@ function readBodyAsBuffer(req: IncomingMessage): Promise { /** * Start an OTEL collector and return it along with the env vars agents need * to export traces to it. + * + * When `customEndpoint` is provided the local in-process collector is NOT + * started. Instead the env vars are set to point at the custom endpoint and + * `collector` is returned as `undefined`. Traces will not be available in the + * web UI in this mode — they are forwarded directly to the external backend. */ -export async function startOtelCollector(persistTracesDir: string): Promise<{ - collector: OtelCollector; +export async function startOtelCollector( + persistTracesDir: string, + customEndpoint?: string +): Promise<{ + collector: OtelCollector | undefined; otelEnvVars: Record; }> { + if (customEndpoint) { + const otelEnvVars: Record = { + OTEL_EXPORTER_OTLP_ENDPOINT: customEndpoint, + OTEL_EXPORTER_OTLP_PROTOCOL: 'http/protobuf', + OTEL_METRICS_EXPORTER: 'none', + AGENT_OBSERVABILITY_ENABLED: 'true', + OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: 'true', + OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED: 'true', + }; + return { collector: undefined, otelEnvVars }; + } + const collector = new OtelCollector({ persistTracesDir }); const collectorPort = await collector.start(); From 06813e10ee6071c4fa45c8caae3331d80ab3f480 Mon Sep 17 00:00:00 2001 From: Massimiliano Angelino Date: Wed, 29 Apr 2026 08:52:56 +0100 Subject: [PATCH 2/3] docs: document --otel-endpoint and --no-traces flags for agentcore dev - docs/commands.md: add --no-traces and --otel-endpoint rows to the dev command flags table - docs/local-development.md: add Telemetry section covering the default local collector, custom OTLP endpoint usage, disabling traces, and trace file storage location --- docs/commands.md | 24 ++++++++++---------- docs/local-development.md | 46 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/docs/commands.md b/docs/commands.md index f6f15b9ae..acd5c4f9d 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -502,17 +502,19 @@ agentcore dev list-tools agentcore dev call-tool --tool myTool --input '{"arg": "value"}' ``` -| Flag / Argument | Description | -| ---------------------- | --------------------------------------------------------------------- | -| `[prompt]` | Send a prompt to a running dev server | -| `-p, --port ` | Port (default: 8080; MCP uses 8000, A2A uses 9000) | -| `-r, --runtime ` | Runtime to run or invoke (required if multiple runtimes) | -| `-s, --stream` | Stream response when invoking | -| `-l, --logs` | Non-interactive stdout logging | -| `--tool ` | MCP tool name (with `call-tool` prompt) | -| `--input ` | MCP tool arguments as JSON (with `--tool`) | -| `-H, --header ` | Custom header (`"Name: Value"`, repeatable) | -| `--exec` | Execute a shell command in the running dev container (Container only) | +| Flag / Argument | Description | +| ------------------------- | ------------------------------------------------------------------------------------------------ | +| `[prompt]` | Send a prompt to a running dev server | +| `-p, --port ` | Port (default: 8080; MCP uses 8000, A2A uses 9000) | +| `-r, --runtime ` | Runtime to run or invoke (required if multiple runtimes) | +| `-s, --stream` | Stream response when invoking | +| `-l, --logs` | Non-interactive stdout logging | +| `--tool ` | MCP tool name (with `call-tool` prompt) | +| `--input ` | MCP tool arguments as JSON (with `--tool`) | +| `-H, --header ` | Custom header (`"Name: Value"`, repeatable) | +| `--exec` | Execute a shell command in the running dev container (Container only) | +| `--no-traces` | Disable local OTEL trace collection entirely | +| `--otel-endpoint ` | Forward agent traces to a custom OTLP/HTTP endpoint instead of the local collector | ### invoke diff --git a/docs/local-development.md b/docs/local-development.md index d13033768..bf0d5a292 100644 --- a/docs/local-development.md +++ b/docs/local-development.md @@ -109,6 +109,52 @@ See [Container Builds](container-builds.md) for full details on container develo Memory requires deployment to test fully. For local testing, you can mock these dependencies in your agent code. +## Telemetry + +In dev mode the CLI automatically starts a local OTLP/HTTP collector and injects the standard OpenTelemetry environment +variables into the agent process. Traces are persisted to `agentcore/.cli/traces/` and displayed in the web UI. + +### Custom OTLP Endpoint + +To forward traces to your own backend (Jaeger, Grafana Tempo, a cloud collector, etc.) instead of the local collector: + +```bash +agentcore dev --otel-endpoint http://localhost:4318 +``` + +When `--otel-endpoint` is set: +- The local in-process collector is **not** started — no port is bound by the CLI +- `OTEL_EXPORTER_OTLP_ENDPOINT` is set to your URL +- All other OTEL env vars are injected as normal +- Traces will **not** appear in the web UI traces panel (they go directly to your backend) + +The endpoint at startup is printed to stdout: + +``` +OTEL traces → http://localhost:4318 +``` + +### Disabling Traces + +To disable telemetry collection entirely: + +```bash +agentcore dev --no-traces +``` + +No OTEL env vars are injected into the agent process. + +### Trace Storage + +When using the default local collector, traces are persisted as OTLP JSON Lines files: + +``` +agentcore/.cli/traces/otlp/ +└── -.otlp.jsonl +``` + +Traces survive dev server restarts and are available in the web UI traces panel across sessions. + ## Gateway Environment Variables When you have deployed gateways, `agentcore dev` automatically injects gateway environment variables into your local From 8620eeff8b7eda48cd9a67ab5894f547a2aed1c7 Mon Sep 17 00:00:00 2001 From: Avi Alpert Date: Thu, 30 Apr 2026 09:08:07 -0400 Subject: [PATCH 3/3] style: fix prettier formatting in docs --- docs/commands.md | 26 +++++++++++++------------- docs/local-development.md | 1 + 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/commands.md b/docs/commands.md index acd5c4f9d..aa50bd6ff 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -502,19 +502,19 @@ agentcore dev list-tools agentcore dev call-tool --tool myTool --input '{"arg": "value"}' ``` -| Flag / Argument | Description | -| ------------------------- | ------------------------------------------------------------------------------------------------ | -| `[prompt]` | Send a prompt to a running dev server | -| `-p, --port ` | Port (default: 8080; MCP uses 8000, A2A uses 9000) | -| `-r, --runtime ` | Runtime to run or invoke (required if multiple runtimes) | -| `-s, --stream` | Stream response when invoking | -| `-l, --logs` | Non-interactive stdout logging | -| `--tool ` | MCP tool name (with `call-tool` prompt) | -| `--input ` | MCP tool arguments as JSON (with `--tool`) | -| `-H, --header ` | Custom header (`"Name: Value"`, repeatable) | -| `--exec` | Execute a shell command in the running dev container (Container only) | -| `--no-traces` | Disable local OTEL trace collection entirely | -| `--otel-endpoint ` | Forward agent traces to a custom OTLP/HTTP endpoint instead of the local collector | +| Flag / Argument | Description | +| ----------------------- | ---------------------------------------------------------------------------------- | +| `[prompt]` | Send a prompt to a running dev server | +| `-p, --port ` | Port (default: 8080; MCP uses 8000, A2A uses 9000) | +| `-r, --runtime ` | Runtime to run or invoke (required if multiple runtimes) | +| `-s, --stream` | Stream response when invoking | +| `-l, --logs` | Non-interactive stdout logging | +| `--tool ` | MCP tool name (with `call-tool` prompt) | +| `--input ` | MCP tool arguments as JSON (with `--tool`) | +| `-H, --header ` | Custom header (`"Name: Value"`, repeatable) | +| `--exec` | Execute a shell command in the running dev container (Container only) | +| `--no-traces` | Disable local OTEL trace collection entirely | +| `--otel-endpoint ` | Forward agent traces to a custom OTLP/HTTP endpoint instead of the local collector | ### invoke diff --git a/docs/local-development.md b/docs/local-development.md index bf0d5a292..5ac0772e0 100644 --- a/docs/local-development.md +++ b/docs/local-development.md @@ -123,6 +123,7 @@ agentcore dev --otel-endpoint http://localhost:4318 ``` When `--otel-endpoint` is set: + - The local in-process collector is **not** started — no port is bound by the CLI - `OTEL_EXPORTER_OTLP_ENDPOINT` is set to your URL - All other OTEL env vars are injected as normal