feat: add structured failure telemetry

packages/argent-cli/package.json

@@ -12,6 +12,7 @@
     "typecheck:tests": "tsc --noEmit -p tsconfig.test.json"
   },
   "dependencies": {
+    "@argent/registry": "file:../registry",
     "@argent/telemetry": "file:../telemetry",
     "@argent/tools-client": "file:../argent-tools-client",
     "@clack/prompts": "^1.1.0",

packages/argent-cli/src/run.ts

@@ -1,6 +1,8 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { createToolsClient, type ToolMeta, type ToolsServerPaths } from "@argent/tools-client";
+import { init as telemetryInit, shutdown as telemetryShutdown, track } from "@argent/telemetry";
+import { FAILURE_CODES, type FailureCode, type FailureKind } from "@argent/registry";
 import {
   parseFlags,
   formatSchemaUsage,
@@ -109,7 +111,33 @@ function renderResult(result: unknown, outputHint: string | undefined, json: boo
   return JSON.stringify(result, null, 2);
 }
 
+const SAFE_TOOL_RE = /^[a-z][a-z0-9-]{0,63}$/;
+
+function safeToolName(toolName: string | undefined): string {
+  return toolName && SAFE_TOOL_RE.test(toolName) ? toolName : "unknown";
+}
+
+async function trackRunFailure(
+  toolName: string | undefined,
+  startedAt: number,
+  signal: {
+    error_code: FailureCode;
+    failure_stage: string;
+    failure_area: "cli";
+    error_kind: FailureKind;
+  }
+): Promise<void> {
+  track("cli:run_fail", {
+    tool: safeToolName(toolName),
+    duration_ms: performance.now() - startedAt,
+    ...signal,
+  });
+  await telemetryShutdown();
+}
+
 export async function run(argv: string[], options: RunCommandOptions): Promise<void> {
+  telemetryInit("cli");
+  const startedAt = performance.now();
   const { fetchTool, callTool } = createToolsClient({ paths: options.paths });
   const [toolName, ...rest] = argv;
 
@@ -134,6 +162,12 @@ Examples:
   const meta = await fetchTool(toolName);
   if (!meta) {
     console.error(`Tool "${toolName}" not found. Run \`argent tools\` to list available tools.`);
+    await trackRunFailure(toolName, startedAt, {
+      error_code: FAILURE_CODES.CLI_RUN_TOOL_NOT_FOUND,
+      failure_stage: "cli_run_fetch_tool",
+      failure_area: "cli",
+      error_kind: "not_found",
+    });
     process.exit(1);
   }
 
@@ -144,6 +178,12 @@ Examples:
     if (err instanceof FlagParseException) {
       console.error(`Error: ${err.message}\n`);
       printToolHelp(meta);
+      await trackRunFailure(toolName, startedAt, {
+        error_code: FAILURE_CODES.CLI_RUN_FLAG_PARSE_FAILED,
+        failure_stage: "cli_run_parse_flags",
+        failure_area: "cli",
+        error_kind: "validation",
+      });
       process.exit(2);
     }
     throw err;
@@ -167,11 +207,23 @@ Examples:
       const parsedRaw = JSON.parse(rawJson);
       if (parsedRaw === null || typeof parsedRaw !== "object" || Array.isArray(parsedRaw)) {
         console.error("--args must be a JSON object");
+        await trackRunFailure(toolName, startedAt, {
+          error_code: FAILURE_CODES.CLI_RUN_ARGS_NOT_OBJECT,
+          failure_stage: "cli_run_parse_raw_args",
+          failure_area: "cli",
+          error_kind: "validation",
+        });
         process.exit(2);
       }
       payload = parsedRaw as Record<string, unknown>;
     } catch (err) {
       console.error(`--args is not valid JSON: ${err instanceof Error ? err.message : err}`);
+      await trackRunFailure(toolName, startedAt, {
+        error_code: FAILURE_CODES.CLI_RUN_ARGS_JSON_INVALID,
+        failure_stage: "cli_run_parse_raw_args",
+        failure_area: "cli",
+        error_kind: "validation",
+      });
       process.exit(2);
     }
   }
@@ -187,6 +239,12 @@ Examples:
     note = resp.note;
   } catch (err) {
     console.error(err instanceof Error ? err.message : String(err));
+    await trackRunFailure(toolName, startedAt, {
+      error_code: FAILURE_CODES.CLI_RUN_TOOL_CALL_FAILED,
+      failure_stage: "cli_run_call_tool",
+      failure_area: "cli",
+      error_kind: "unknown",
+    });
     process.exit(1);
   }
 
@@ -197,6 +255,12 @@ Examples:
       await fetchImageToFile(result as { url?: string; path?: string }, outPath);
     } catch (err) {
       console.error(`Failed to save image: ${err instanceof Error ? err.message : err}`);
+      await trackRunFailure(toolName, startedAt, {
+        error_code: FAILURE_CODES.CLI_RUN_SAVE_IMAGE_FAILED,
+        failure_stage: "cli_run_save_image",
+        failure_area: "cli",
+        error_kind: "unknown",
+      });
       process.exit(1);
     }
   }

packages/argent-cli/test/run-telemetry.test.ts

@@ -0,0 +1,91 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { run } from "../src/run.js";
+
+const toolsClientMock = vi.hoisted(() => ({
+  fetchTool: vi.fn(),
+  callTool: vi.fn(),
+}));
+
+const telemetryMock = vi.hoisted(() => ({
+  init: vi.fn(),
+  shutdown: vi.fn(async () => undefined),
+  track: vi.fn(),
+}));
+
+vi.mock("@argent/tools-client", () => ({
+  createToolsClient: vi.fn(() => toolsClientMock),
+}));
+
+vi.mock("@argent/telemetry", () => telemetryMock);
+
+const toolMeta = {
+  name: "sample-tool",
+  description: "Sample tool",
+  inputSchema: {
+    type: "object",
+    properties: {},
+  },
+};
+
+describe("argent run telemetry", () => {
+  let exitSpy: ReturnType<typeof vi.spyOn>;
+  let errorSpy: ReturnType<typeof vi.spyOn>;
+  let logSpy: ReturnType<typeof vi.spyOn>;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    toolsClientMock.fetchTool.mockResolvedValue(toolMeta);
+    toolsClientMock.callTool.mockResolvedValue({ data: { ok: true } });
+    exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`process.exit:${code}`);
+    }) as typeof process.exit);
+    errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
+    logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined);
+  });
+
+  afterEach(() => {
+    exitSpy.mockRestore();
+    errorSpy.mockRestore();
+    logSpy.mockRestore();
+  });
+
+  it("emits cli:run_fail, not tool:fail, when tool-server call fails", async () => {
+    toolsClientMock.callTool.mockRejectedValue(new Error("server already tracked this"));
+
+    await expect(run(["sample-tool"], { paths: {} as never })).rejects.toThrow("process.exit:1");
+
+    expect(console.error).toHaveBeenCalledWith("server already tracked this");
+    expect(telemetryMock.track).toHaveBeenCalledWith(
+      "cli:run_fail",
+      expect.objectContaining({
+        tool: "sample-tool",
+        error_code: "CLI_RUN_TOOL_CALL_FAILED",
+        failure_stage: "cli_run_call_tool",
+        failure_area: "cli",
+        error_kind: "unknown",
+      })
+    );
+    expect(telemetryMock.track).not.toHaveBeenCalledWith("tool:fail", expect.anything());
+    expect(telemetryMock.shutdown).toHaveBeenCalledTimes(1);
+  });
+
+  it("emits cli:run_fail, not tool:fail, for local CLI argument parsing failures", async () => {
+    await expect(
+      run(["sample-tool", "--args", "not-json"], { paths: {} as never })
+    ).rejects.toThrow("process.exit:2");
+
+    expect(toolsClientMock.callTool).not.toHaveBeenCalled();
+    expect(telemetryMock.track).toHaveBeenCalledWith(
+      "cli:run_fail",
+      expect.objectContaining({
+        tool: "sample-tool",
+        error_code: "CLI_RUN_ARGS_JSON_INVALID",
+        failure_stage: "cli_run_parse_raw_args",
+        failure_area: "cli",
+        error_kind: "validation",
+      })
+    );
+    expect(telemetryMock.track).not.toHaveBeenCalledWith("tool:fail", expect.anything());
+    expect(telemetryMock.shutdown).toHaveBeenCalledTimes(1);
+  });
+});

packages/argent-installer/package.json

@@ -12,6 +12,7 @@
     "typecheck:tests": "tsc --noEmit -p tsconfig.test.json"
   },
   "dependencies": {
+    "@argent/registry": "file:../registry",
     "@argent/telemetry": "file:../telemetry",
     "@argent/tools-client": "file:../argent-tools-client",
     "@argent/update-core": "file:../update-core",