Add Datadog metrics for MPP Discovery Service MCP

scripts/check-mpp-discovery-service-mcp.mjs

@@ -6,7 +6,6 @@ const DEFAULT_ENDPOINT = "https://mpp.dev/mcp/services";
 const endpoint = normalizeEndpoint(
   process.env.MPP_DISCOVERY_MCP_URL ?? DEFAULT_ENDPOINT,
 );
-const slackWebhookUrl = process.env.SLACK_WEBHOOK_URL;
 const requestTimeoutMs = Number(
   process.env.MPP_DISCOVERY_MCP_TIMEOUT_MS ?? 30_000,
 );
@@ -317,7 +316,6 @@ async function main() {
     return "invalid enum rejected";
   });
 
-  await reportToSlack();
   await writeStepSummary();
 
   const failed = results.filter((result) => !result.ok);
@@ -524,41 +522,6 @@ async function writeStepSummary() {
   await appendFile(process.env.GITHUB_STEP_SUMMARY, summary);
 }
 
-async function reportToSlack() {
-  if (!slackWebhookUrl) {
-    console.log("SLACK_WEBHOOK_URL is not set; skipping Slack report.");
-    return;
-  }
-  const startedAt = Date.now();
-  try {
-    const response = await fetchWithTimeout(slackWebhookUrl, {
-      method: "POST",
-      headers: { "content-type": "application/json" },
-      body: JSON.stringify({ text: buildSummaryText() }),
-    });
-    if (!response.ok) {
-      const text = await response.text();
-      results.push({
-        name: "Slack report",
-        ok: false,
-        detail: `Slack webhook returned HTTP ${response.status}: ${snippet(text)}`,
-        durationMs: Date.now() - startedAt,
-      });
-      console.error(results.at(-1).detail);
-      return;
-    }
-    console.log("ok - Slack report");
-  } catch (error) {
-    results.push({
-      name: "Slack report",
-      ok: false,
-      detail: errorMessage(error),
-      durationMs: Date.now() - startedAt,
-    });
-    console.error(results.at(-1).detail);
-  }
-}
-
 main().catch((error) => {
   console.error(errorMessage(error));
   process.exitCode = 1;

workers/mcp-services/README.md

@@ -39,13 +39,62 @@ authoritative.
 - KV binding: `MPP_CATALOG_CACHE`
 - Cache key: `mpp:services:v1`
 - Hourly cron: `0 * * * *`
+- Health cron: `* * * * *`
 - Requests use fresh KV data when it is less than one hour old.
 - If KV data is stale, requests serve the last-good cached catalog and refresh
   in the background.
 - If `mpp.dev` is unreachable during cron refresh, the Worker logs the failure
   and keeps the last-good KV value.
 - There is no public write, sync, registration, payment, or auth path.
 
+## Datadog monitoring
+
+The Worker emits custom Datadog metrics directly from production. Runtime
+request metrics are emitted with `ctx.waitUntil()` so user-facing MCP responses
+do not wait on Datadog ingestion.
+
+The one-minute health cron calls the public endpoint
+`https://mpp.dev/mcp/services`, then checks:
+
+- `GET /mcp/services`
+- `HEAD /mcp/services`
+- JSON-RPC `initialize`
+- JSON-RPC `tools/list`
+- JSON-RPC `tools/call` for `get_catalog_status`
+- JSON-RPC `tools/call` for `search_services`
+
+Metrics use the `mpp.discovery_mcp.*` namespace. Important metrics include:
+
+- `mpp.discovery_mcp.http.request.count`
+- `mpp.discovery_mcp.http.response.duration_ms`
+- `mpp.discovery_mcp.http.error.count`
+- `mpp.discovery_mcp.health.ok`
+- `mpp.discovery_mcp.health.check.ok`
+- `mpp.discovery_mcp.health.check.duration_ms`
+- `mpp.discovery_mcp.catalog.services`
+- `mpp.discovery_mcp.catalog.offers`
+- `mpp.discovery_mcp.catalog.cache_age_seconds`
+- `mpp.discovery_mcp.catalog.refresh.ok`
+- `mpp.discovery_mcp.catalog.refresh.duration_ms`
+
+Production Worker vars:
+
+```text
+DATADOG_ENABLED=true
+DATADOG_ENV=production
+DATADOG_SERVICE=mpp-discovery-service-mcp
+DATADOG_SITE=us5.datadoghq.com
+```
+
+Production Worker secret:
+
+```text
+DATADOG_API_KEY=<datadog-api-key>
+```
+
+Only the Worker metrics API key is required by this package. Configure Datadog
+notifications manually from these emitted metrics.
+
 ## Tools
 
 All tool responses include `structuredContent`, an `outputSchema`, and a text

workers/mcp-services/src/cache.ts

@@ -1,4 +1,4 @@
-import type { ServicesResponse } from "./types.js";
+import type { ServicesResponse, WorkerEnv } from "./types.js";
 
 const CACHE_KEY = "mpp:services:v1";
 const CACHE_MAX_AGE_MS = 60 * 60 * 1000;
@@ -15,7 +15,7 @@ export type CatalogSnapshot = CachedCatalog & {
 };
 
 export async function getCatalog(
-  env: Env,
+  env: WorkerEnv,
   ctx?: ExecutionContext,
 ): Promise<CatalogSnapshot> {
   const cached = await readCachedCatalog(env);
@@ -39,7 +39,7 @@ export async function getCatalog(
   return refreshCatalog(env);
 }
 
-export async function refreshCatalog(env: Env): Promise<CatalogSnapshot> {
+export async function refreshCatalog(env: WorkerEnv): Promise<CatalogSnapshot> {
   const sourceUrl = env.MPP_SERVICES_URL || DEFAULT_SERVICES_URL;
   const catalog = await fetchCatalog(sourceUrl);
   const cached: CachedCatalog = {
@@ -51,7 +51,9 @@ export async function refreshCatalog(env: Env): Promise<CatalogSnapshot> {
   return { ...cached, cacheStatus: "refreshed" };
 }
 
-async function readCachedCatalog(env: Env): Promise<CachedCatalog | undefined> {
+async function readCachedCatalog(
+  env: WorkerEnv,
+): Promise<CachedCatalog | undefined> {
   const cached = await env.MPP_CATALOG_CACHE.get(CACHE_KEY, "json");
   if (isCachedCatalog(cached)) return cached;
   return undefined;

workers/mcp-services/src/datadog.test.ts

@@ -0,0 +1,75 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { datadogEnabled, gauge, postMetrics } from "./datadog.js";
+import type { WorkerEnv } from "./types.js";
+
+describe("datadog metrics", () => {
+  afterEach(() => {
+    vi.unstubAllGlobals();
+  });
+
+  it("posts metrics to the configured Datadog site", async () => {
+    const requests: Request[] = [];
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async (input: RequestInfo, init?: RequestInit) => {
+        requests.push(new Request(input, init));
+        return new Response("{}", { status: 202 });
+      }),
+    );
+
+    await postMetrics(env(), [gauge("health.ok", 1, ["check:initialize"])]);
+
+    expect(requests).toHaveLength(1);
+    expect(requests[0].url).toBe("https://api.us5.datadoghq.com/api/v1/series");
+    expect(requests[0].headers.get("DD-API-KEY")).toBe("dd-key");
+    const body = (await requests[0].json()) as {
+      series: Array<{
+        metric: string;
+        points: [[number, number]];
+        tags: string[];
+      }>;
+    };
+    expect(body.series[0]).toEqual(
+      expect.objectContaining({
+        metric: "mpp.discovery_mcp.health.ok",
+        points: [[expect.any(Number), 1]],
+        tags: [
+          "service:mpp-discovery-service-mcp",
+          "env:production",
+          "check:initialize",
+        ],
+      }),
+    );
+  });
+
+  it("skips posting when disabled", async () => {
+    const fetch = vi.fn();
+    vi.stubGlobal("fetch", fetch);
+
+    await postMetrics(
+      {
+        ...env(),
+        DATADOG_ENABLED: "false",
+      },
+      [gauge("health.ok", 1)],
+    );
+
+    expect(fetch).not.toHaveBeenCalled();
+  });
+
+  it("requires explicit enablement or an API key", () => {
+    expect(datadogEnabled({} as WorkerEnv)).toBe(false);
+    expect(datadogEnabled({ DATADOG_ENABLED: "true" } as WorkerEnv)).toBe(true);
+    expect(datadogEnabled({ DATADOG_API_KEY: "key" } as WorkerEnv)).toBe(true);
+  });
+});
+
+function env(): WorkerEnv {
+  return {
+    DATADOG_API_KEY: "dd-key",
+    DATADOG_ENABLED: "true",
+    DATADOG_ENV: "production",
+    DATADOG_SERVICE: "mpp-discovery-service-mcp",
+    DATADOG_SITE: "us5.datadoghq.com",
+  } as WorkerEnv;
+}

workers/mcp-services/src/datadog.ts

@@ -0,0 +1,112 @@
+import type { WorkerEnv } from "./types.js";
+
+type MetricType = "count" | "gauge";
+
+export type MetricPoint = {
+  metric: string;
+  value: number;
+  type: MetricType;
+  tags?: string[];
+};
+
+const DEFAULT_DATADOG_SITE = "us5.datadoghq.com";
+const METRIC_PREFIX = "mpp.discovery_mcp";
+const DEFAULT_SERVICE = "mpp-discovery-service-mcp";
+const DEFAULT_ENV = "production";
+
+export function gauge(
+  name: string,
+  value: number,
+  tags?: string[],
+): MetricPoint {
+  return { metric: `${METRIC_PREFIX}.${name}`, type: "gauge", value, tags };
+}
+
+export function count(name: string, value = 1, tags?: string[]): MetricPoint {
+  return { metric: `${METRIC_PREFIX}.${name}`, type: "count", value, tags };
+}
+
+export function queueMetrics(
+  ctx: ExecutionContext,
+  env: WorkerEnv,
+  metrics: MetricPoint[],
+): void {
+  if (!datadogEnabled(env) || metrics.length === 0) return;
+
+  ctx.waitUntil(
+    postMetrics(env, metrics).catch((error) => {
+      console.error(
+        JSON.stringify({
+          message: "datadog.metrics_failed",
+          error: errorMessage(error),
+        }),
+      );
+    }),
+  );
+}
+
+export async function postMetrics(
+  env: WorkerEnv,
+  metrics: MetricPoint[],
+): Promise<void> {
+  if (!datadogEnabled(env) || metrics.length === 0) return;
+
+  if (!env.DATADOG_API_KEY) {
+    console.warn(
+      JSON.stringify({
+        message: "datadog.metrics_skipped",
+        reason: "missing_api_key",
+      }),
+    );
+    return;
+  }
+
+  const timestamp = Math.floor(Date.now() / 1000);
+  const baseTags = [
+    `service:${env.DATADOG_SERVICE || DEFAULT_SERVICE}`,
+    `env:${env.DATADOG_ENV || DEFAULT_ENV}`,
+  ];
+  const response = await fetch(`${apiBase(env)}/api/v1/series`, {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+      "DD-API-KEY": env.DATADOG_API_KEY,
+    },
+    body: JSON.stringify({
+      series: metrics.map((metric) => ({
+        metric: metric.metric,
+        type: metric.type,
+        points: [[timestamp, metric.value]],
+        tags: [...baseTags, ...(metric.tags ?? [])],
+      })),
+    }),
+  });
+
+  if (!response.ok) {
+    throw new Error(`Datadog metrics API failed: ${response.status}`);
+  }
+}
+
+export function datadogEnabled(env: WorkerEnv): boolean {
+  const configured = normalized(env.DATADOG_ENABLED);
+  if (configured === "true") return true;
+  if (configured === "false") return false;
+  return Boolean(env.DATADOG_API_KEY);
+}
+
+function apiBase(env: WorkerEnv): string {
+  const site = env.DATADOG_SITE || DEFAULT_DATADOG_SITE;
+  if (/^https?:\/\//i.test(site)) return site.replace(/\/+$/, "");
+  if (site.startsWith("api.")) return `https://${site}`;
+  return `https://api.${site}`;
+}
+
+function normalized(value: string | undefined): string {
+  return String(value ?? "")
+    .trim()
+    .toLowerCase();
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}