test: check the real API, and detect the spec drifting under a published version

.github/workflows/live.yml

@@ -0,0 +1,37 @@
+name: live
+
+# Never on pull_request. These tests spend money on every run, and a pull request
+# from a fork cannot read secrets, so it would fail for everyone outside the org.
+on:
+  workflow_dispatch:
+  schedule:
+    # Mondays, 13:00 UTC. The point of a schedule is to notice the API moving
+    # under a published version: the contract suite reads a pinned snapshot and
+    # only moves when somebody runs `pnpm spec:refresh`, so nothing else fails
+    # when Tako changes and this repo does not.
+    - cron: "0 13 * * 1"
+
+# One run at a time. Concurrent runs on the same key hit Tako's throttle and
+# report a rate limit as a contract failure.
+concurrency:
+  group: live-api
+  cancel-in-progress: false
+
+jobs:
+  live:
+    runs-on: ubuntu-latest
+    # Absent secret means an unrunnable job, which should not read as a failure.
+    if: ${{ github.event_name == 'workflow_dispatch' || vars.LIVE_TESTS_ENABLED == 'true' }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm test:live
+        env:
+          TAKO_API_KEY: ${{ secrets.TAKO_API_KEY }}
+          # Optional. Leave unset to test the same host the SDK defaults to.
+          TAKO_BASE_URL: ${{ vars.TAKO_BASE_URL }}

CONTRIBUTING.md

@@ -10,6 +10,7 @@ pnpm typecheck     # tsc over src + tests + examples
 pnpm build         # tsup → dist/
 pnpm lint:package  # publint + are-the-types-wrong, against the packed tarball
 pnpm test:package  # install the tarball in a scratch project and use it
+pnpm test:live     # real API calls — needs TAKO_API_KEY, costs money
 pnpm spec:refresh  # re-vendor tests/contract/openapi.yaml from docs.tako.com
 ```
 
@@ -48,6 +49,41 @@ every `.d.ts` under `node_modules`, and the `ai` package's own tree reports
 missing `@types/node` and `@types/json-schema` — another package's noise, loud
 enough to hide a real failure here.
 
+## Checking the API itself (`tests/live/`)
+
+Every other suite proves a request body is **legal** against a vendored snapshot of
+the OpenAPI document. None of them prove the API **honors** the option, or that the
+snapshot still matches reality — the gap that let the 2.x types rot for two months
+while every test stayed green.
+
+`tests/live/` closes it from the other side. It sends real requests and validates
+the responses with the same ajv validators the contract suite uses, so **a response
+that stops matching the vendored spec fails even though nothing in this repo
+changed.** That makes it the upstream drift detector the parity tests cannot be:
+those read a pinned snapshot and only move when someone runs `pnpm spec:refresh`.
+
+```bash
+TAKO_API_KEY=... pnpm test:live
+TAKO_API_KEY=... TAKO_BASE_URL=https://some-other-host pnpm test:live   # optional
+```
+
+- **Excluded from `pnpm test`** by `vitest.config.ts`, and only included by
+  `vitest.live.config.ts`. It costs money, so it must never run by accident.
+- **Skips without a key** rather than failing, so a contributor with no key sees
+  no red.
+- **Runs on a schedule** (`.github/workflows/live.yml`, Mondays 13:00 UTC) plus
+  manual dispatch. Never on `pull_request`: forks cannot read secrets, so it
+  would fail for every outside contributor.
+- Serial, with one retry, because the tests compare responses across requests and
+  Tako throttles per key.
+
+Two rules for anything you add there:
+
+1. **Assert contract, never content.** "A card came back" is stable. "The first
+   card is Nvidia revenue" is one ranking change from a false alarm.
+2. **Never trigger a billed export.** `quoteOnly` prices one for free, and that is
+   the only way this suite touches export pricing.
+
 ## Keeping the API contract honest
 
 `tests/contract/` checks this SDK's types against two pinned references: the

package.json

@@ -22,8 +22,9 @@
     "test": "vitest run",
     "test:watch": "vitest",
     "test:contract": "vitest run tests/contract",
+    "test:live": "vitest run --config vitest.live.config.ts",
     "test:package": "node scripts/verify-package.mjs",
-    "lint:package": "npx -y publint@latest --strict --pack npm && npx -y @arethetypeswrong/cli@latest --pack . --ignore-rules cjs-resolves-to-esm",
+    "lint:package": "npm run --silent build && npx -y publint@latest --strict --pack npm && npx -y @arethetypeswrong/cli@latest --pack . --ignore-rules cjs-resolves-to-esm",
     "spec:refresh": "curl -fsSL https://docs.tako.com/api-reference/openapi.yaml -o tests/contract/openapi.yaml",
     "prepublishOnly": "pnpm build"
   },

tests/live/options.live.test.ts

@@ -0,0 +1,212 @@
+/**
+ * Live checks against the real Tako API.
+ *
+ * Every other suite in this repo proves a request body is *legal* against a
+ * vendored snapshot of the OpenAPI document. None of them prove the API honors
+ * the option, or that the snapshot still matches reality. That gap is why the
+ * 2.x types rotted for two months while 18 tests stayed green.
+ *
+ * These tests close it from the other side: they send real requests and validate
+ * the responses with the same ajv validators the contract suite uses, so a
+ * response that stops matching the vendored spec fails here even though nothing
+ * in this repo changed. That makes this the upstream drift detector the parity
+ * tests cannot be — those read a pinned snapshot and only move when a human runs
+ * `pnpm spec:refresh`.
+ *
+ * Excluded from `pnpm test`. Run with `pnpm test:live` and a key, or let the
+ * `live` workflow run it on a schedule. Without `TAKO_API_KEY` every test skips
+ * rather than fails, so a contributor with no key sees no red.
+ *
+ * Two rules for anything added here:
+ *
+ * 1. Assert contract, never content. "A card came back" is stable; "the first
+ *    card is Nvidia revenue" is a ranking change away from a false alarm.
+ * 2. Never trigger a billed export. `quoteOnly` prices one for free, and that is
+ *    the only way this file touches export pricing.
+ */
+import { describe, expect, it } from "vitest";
+import { check } from "../contract/spec";
+import { buildContentsRequestBody, buildSearchRequestBody } from "../../src/request";
+import type { TakoContentsConfig, TakoRetrievalConfig } from "../../src/types";
+
+const KEY = process.env.TAKO_API_KEY ?? process.env.TAKO_API_TOKEN;
+
+// Defaults to the same host the SDK does. Set TAKO_BASE_URL to point at another
+// environment; no hostname other than the public default is committed here.
+const BASE = (process.env.TAKO_BASE_URL ?? "https://tako.com").replace(/\/+$/, "");
+
+const TIMEOUT = 60_000;
+
+async function post(path: string, body: unknown) {
+  const response = await fetch(`${BASE}${path}`, {
+    method: "POST",
+    headers: { "X-API-Key": KEY as string, "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+  });
+  const text = await response.text();
+  let json: Record<string, unknown> | null = null;
+  try {
+    json = JSON.parse(text) as Record<string, unknown>;
+  } catch {
+    // A non-JSON body is itself the failure; the assertion reports `text`.
+  }
+  return { status: response.status, json, text };
+}
+
+const search = (config: TakoRetrievalConfig, query = "nvidia revenue") =>
+  post("/api/v3/search", buildSearchRequestBody(config, query));
+
+const hostsOf = (json: Record<string, unknown> | null) =>
+  ((json?.web_results as { url: string }[] | undefined) ?? []).map((w) => {
+    try {
+      return new URL(w.url).hostname.replace(/^www\./, "");
+    } catch {
+      return "";
+    }
+  });
+
+describe.skipIf(!KEY)("live: the request options reach a real API", () => {
+  it(
+    "a query-only body is accepted, and matches one that restates the old defaults",
+    async () => {
+      // This is why the check exists. 3.0.0 stopped sending effort, country_code
+      // and locale, on the evidence that the spec documents exactly the values
+      // the SDK used to hardcode. "The document says X is the default" and "the
+      // API applies X when the key is absent" are different claims, and only
+      // this one tests the second.
+      const bare = buildSearchRequestBody({}, "nvidia revenue");
+      expect(Object.keys(bare)).toEqual(["query"]);
+
+      const [a, b] = await Promise.all([
+        search({}),
+        search({ effort: "fast", countryCode: "US", locale: "en-US" }),
+      ]);
+
+      expect(a.status, a.text.slice(0, 300)).toBe(200);
+      expect(b.status, b.text.slice(0, 300)).toBe(200);
+      expect(Object.keys(a.json ?? {}).sort()).toEqual(Object.keys(b.json ?? {}).sort());
+      expect((a.json?.web_results as unknown[] | undefined)?.length).toBe(
+        (b.json?.web_results as unknown[] | undefined)?.length,
+      );
+    },
+    TIMEOUT,
+  );
+
+  // A rejected option is an HTTP 400 with "Extra inputs are not permitted",
+  // which is how `deferDataRetrieval` failed for two months while typed as valid.
+  const accepted: [string, TakoRetrievalConfig][] = [
+    ["location", { location: { latitude: 37.77, longitude: -122.42 } }],
+    ["data.contentFormat", { sources: { data: { includeContents: true, contentFormat: "json_records" } } }],
+    ["data.mode", { sources: { data: { includeContents: true, mode: "inline" } } }],
+    ["web.category", { sources: { web: { category: "news" } } }],
+    ["web.includeDomains", { sources: { web: { includeDomains: ["reuters.com"] } } }],
+    ["web.excludeDomains", { sources: { web: { excludeDomains: ["reddit.com"] } } }],
+    ["web.snippetMaxChars", { sources: { web: { snippetMaxChars: 300 } } }],
+    ["web.articleContentMaxChars", { sources: { web: { includeContents: true, articleContentMaxChars: 5000 } } }],
+    ["web.publishedAfter", { sources: { web: { publishedAfter: "2026-01-01" } } }],
+    ["web.publishedBefore", { sources: { web: { publishedBefore: "2026-12-31" } } }],
+  ];
+
+  it.each(accepted)("the API accepts %s", async (_name, config) => {
+    const r = await search(config);
+    expect(r.status, r.text.slice(0, 300)).toBe(200);
+  }, TIMEOUT);
+
+  it(
+    "a live search response still validates against the vendored spec",
+    async () => {
+      // The drift detector. Nothing in this repo has to change for this to fail —
+      // it fails when the API stops matching the snapshot the other suites trust.
+      const r = await search({ sources: { data: { includeContents: true }, web: { count: 3 } } });
+      expect(r.status).toBe(200);
+      expect(check("SearchResponse", r.json).errors).toEqual([]);
+    },
+    TIMEOUT,
+  );
+
+  it(
+    "includeDomains and excludeDomains actually filter",
+    async () => {
+      const only = await search(
+        { sources: { web: { includeDomains: ["reuters.com"], count: 5 } } },
+        "nvidia earnings",
+      );
+      expect(only.status).toBe(200);
+      const kept = hostsOf(only.json);
+      // An empty list means the filter applied and nothing matched, which is a
+      // pass for "it filters" — a leak is the failure.
+      expect(kept.filter((h) => h && !h.endsWith("reuters.com"))).toEqual([]);
+
+      const unfiltered = await search({ sources: { web: { count: 5 } } }, "nvidia earnings");
+      const drop = hostsOf(unfiltered.json)[0];
+      if (drop) {
+        const without = await search(
+          { sources: { web: { excludeDomains: [drop], count: 5 } } },
+          "nvidia earnings",
+        );
+        expect(without.status).toBe(200);
+        expect(hostsOf(without.json)).not.toContain(drop);
+      }
+    },
+    TIMEOUT * 2,
+  );
+
+  it.each([
+    ["json_records", "records"],
+    ["json_compact", "dataset"],
+  ] as const)(
+    "contentFormat %s is honored and populates %s",
+    async (format, field) => {
+      const r = await search({
+        sources: { data: { includeContents: true, contentFormat: format, count: 1 } },
+      });
+      expect(r.status).toBe(200);
+      const content = (r.json?.cards as { content?: Record<string, unknown> }[] | undefined)?.[0]
+        ?.content;
+      if (!content) return; // No inlined card to inspect; nothing to assert.
+      expect(content.content_format).toBe(format);
+      expect(content[field]).not.toBeNull();
+    },
+    TIMEOUT,
+  );
+});
+
+describe.skipIf(!KEY)("live: contents pricing, quoted rather than bought", () => {
+  it(
+    "quoteOnly returns a price and no content, and maxRows clamps instead of failing",
+    async () => {
+      const seed = await search({});
+      expect(seed.status).toBe(200);
+      const url = (seed.json?.cards as { webpage_url?: string }[] | undefined)?.[0]?.webpage_url;
+      if (!url) return; // No card to quote against.
+
+      const quote = async (config: TakoContentsConfig) => {
+        const r = await post("/api/v1/contents", buildContentsRequestBody(url, config));
+        expect(r.status, r.text.slice(0, 300)).toBe(200);
+        expect(check("ContentsResponse", r.json).errors).toEqual([]);
+        return (r.json?.contents as Record<string, unknown>[] | undefined)?.[0];
+      };
+
+      const small = await quote({ quoteOnly: true, maxRows: 20 });
+      const large = await quote({ quoteOnly: true, maxRows: 2000 });
+      if (!small || !large) return;
+
+      // A quote carries pricing and withholds content.
+      expect(small.url ?? null).toBeNull();
+      expect(small.data ?? null).toBeNull();
+      expect(small.export_pricing).toBeTruthy();
+
+      // `cost` on a quote is the price the export would be, not a charge — which
+      // is only observable because it scales with maxRows.
+      expect(typeof small.cost).toBe("number");
+      expect(large.cost as number).toBeGreaterThan(small.cost as number);
+
+      // The documented behavior worth a live test: over the ceiling the API
+      // clamps and bills what it returns, so a caller trusting a 400 gets a
+      // short export, a charge, and no error.
+      const over = await quote({ quoteOnly: true, maxRows: 999_999 });
+      if (over) expect(over.cost).toBe(large.cost);
+    },
+    TIMEOUT * 3,
+  );
+});

vitest.config.ts

@@ -1,8 +1,12 @@
-import { defineConfig } from "vitest/config";
+import { configDefaults, defineConfig } from "vitest/config";
 
 export default defineConfig({
   test: {
     environment: "node",
     include: ["tests/**/*.test.ts"],
+    // `tests/live/` makes real, billed API calls and needs a key. It must never
+    // run as part of `pnpm test`, which every contributor and CI job runs.
+    // `vitest.live.config.ts` is the only config that includes it.
+    exclude: [...configDefaults.exclude, "tests/live/**"],
   },
 });

vitest.live.config.ts

@@ -0,0 +1,20 @@
+import { defineConfig } from "vitest/config";
+
+/**
+ * The live suite only. Kept in its own config so `pnpm test` cannot pick these
+ * up: they need a real key and they cost money.
+ *
+ * Serial by design. The tests compare responses across requests, and Tako
+ * throttles per key, so a parallel run makes both the comparisons and the rate
+ * limit unpredictable.
+ */
+export default defineConfig({
+  test: {
+    environment: "node",
+    include: ["tests/live/**/*.test.ts"],
+    testTimeout: 60_000,
+    hookTimeout: 60_000,
+    fileParallelism: false,
+    retry: 1, // one retry absorbs a transient 5xx without hiding a real failure
+  },
+});