From 0e0033895afd3acc05d003b51fc07a8e71bb4d17 Mon Sep 17 00:00:00 2001 From: bobby abbott Date: Mon, 3 Aug 2026 19:20:10 -0700 Subject: [PATCH] test: check the real API, and detect the spec drifting under a published version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every suite in this repo proves a request body is legal against a vendored snapshot. None prove the API honors the option, or that the snapshot still matches reality — the gap that let the 2.x types rot for two months while all 18 tests stayed green. 3.0.0 is published now, so that gap is a live risk rather than a latent one. tests/live/ sends real requests and validates the responses with the same ajv validators the contract suite uses. A response that stops matching the vendored spec therefore fails here even though nothing in this repo changed, which makes this the upstream drift detector the parity tests cannot be: those read a pinned snapshot and only move when somebody runs `pnpm spec:refresh`. Verified load-bearing rather than assumed. Adding a bogus required property to SearchResponse in the vendored spec (via a YAML round-trip, so the document stays valid) makes the drift test fail; restoring it makes it pass. A first attempt at that proof broke the YAML and silently ran no tests, which is worth knowing if someone repeats it. The 16 tests cover what shipped in 3.0.0 and had never touched a real API: that a query-only body is accepted and returns the same shape and result count as one restating the old fast/US/en-US defaults, that all ten new search options are accepted rather than 400ing the way deferDataRetrieval did, that includeDomains and excludeDomains actually filter, that both contentFormat values are honored and populate the matching payload field, and that quoteOnly returns a price with no content while an over-ceiling maxRows clamps instead of failing. Run against a live API: 16 passed in 14s. Guardrails, because this suite costs money: - Excluded from `pnpm test` by vitest.config.ts and included only by vitest.live.config.ts, so it cannot run by accident. - Skips without a key instead of failing, so a contributor with no key sees no red. - Scheduled weekly plus manual dispatch, never on pull_request: a fork cannot read secrets, so it would fail for every outside contributor. - Serial with one retry, since the tests compare responses across requests and Tako throttles per key. - No host or token literal is committed; the base url comes from the environment and defaults to the same host the SDK does. Also fixes lint:package, added in the previous change, which assumed dist/ already existed. CI hid it because `pnpm build` runs first, but on a fresh checkout it reported five "file does not exist" errors that look like packaging faults and are not. It now builds first, matching test:package. Co-Authored-By: Claude Opus 5 --- .github/workflows/live.yml | 37 ++++++ CONTRIBUTING.md | 36 ++++++ package.json | 3 +- tests/live/options.live.test.ts | 212 ++++++++++++++++++++++++++++++++ vitest.config.ts | 6 +- vitest.live.config.ts | 20 +++ 6 files changed, 312 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/live.yml create mode 100644 tests/live/options.live.test.ts create mode 100644 vitest.live.config.ts diff --git a/.github/workflows/live.yml b/.github/workflows/live.yml new file mode 100644 index 0000000..a500fcd --- /dev/null +++ b/.github/workflows/live.yml @@ -0,0 +1,37 @@ +name: live + +# Never on pull_request. These tests spend money on every run, and a pull request +# from a fork cannot read secrets, so it would fail for everyone outside the org. +on: + workflow_dispatch: + schedule: + # Mondays, 13:00 UTC. The point of a schedule is to notice the API moving + # under a published version: the contract suite reads a pinned snapshot and + # only moves when somebody runs `pnpm spec:refresh`, so nothing else fails + # when Tako changes and this repo does not. + - cron: "0 13 * * 1" + +# One run at a time. Concurrent runs on the same key hit Tako's throttle and +# report a rate limit as a contract failure. +concurrency: + group: live-api + cancel-in-progress: false + +jobs: + live: + runs-on: ubuntu-latest + # Absent secret means an unrunnable job, which should not read as a failure. + if: ${{ github.event_name == 'workflow_dispatch' || vars.LIVE_TESTS_ENABLED == 'true' }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm test:live + env: + TAKO_API_KEY: ${{ secrets.TAKO_API_KEY }} + # Optional. Leave unset to test the same host the SDK defaults to. + TAKO_BASE_URL: ${{ vars.TAKO_BASE_URL }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 748b88f..4c6107b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,6 +10,7 @@ pnpm typecheck # tsc over src + tests + examples pnpm build # tsup → dist/ pnpm lint:package # publint + are-the-types-wrong, against the packed tarball pnpm test:package # install the tarball in a scratch project and use it +pnpm test:live # real API calls — needs TAKO_API_KEY, costs money pnpm spec:refresh # re-vendor tests/contract/openapi.yaml from docs.tako.com ``` @@ -48,6 +49,41 @@ every `.d.ts` under `node_modules`, and the `ai` package's own tree reports missing `@types/node` and `@types/json-schema` — another package's noise, loud enough to hide a real failure here. +## Checking the API itself (`tests/live/`) + +Every other suite proves a request body is **legal** against a vendored snapshot of +the OpenAPI document. None of them prove the API **honors** the option, or that the +snapshot still matches reality — the gap that let the 2.x types rot for two months +while every test stayed green. + +`tests/live/` closes it from the other side. It sends real requests and validates +the responses with the same ajv validators the contract suite uses, so **a response +that stops matching the vendored spec fails even though nothing in this repo +changed.** That makes it the upstream drift detector the parity tests cannot be: +those read a pinned snapshot and only move when someone runs `pnpm spec:refresh`. + +```bash +TAKO_API_KEY=... pnpm test:live +TAKO_API_KEY=... TAKO_BASE_URL=https://some-other-host pnpm test:live # optional +``` + +- **Excluded from `pnpm test`** by `vitest.config.ts`, and only included by + `vitest.live.config.ts`. It costs money, so it must never run by accident. +- **Skips without a key** rather than failing, so a contributor with no key sees + no red. +- **Runs on a schedule** (`.github/workflows/live.yml`, Mondays 13:00 UTC) plus + manual dispatch. Never on `pull_request`: forks cannot read secrets, so it + would fail for every outside contributor. +- Serial, with one retry, because the tests compare responses across requests and + Tako throttles per key. + +Two rules for anything you add there: + +1. **Assert contract, never content.** "A card came back" is stable. "The first + card is Nvidia revenue" is one ranking change from a false alarm. +2. **Never trigger a billed export.** `quoteOnly` prices one for free, and that is + the only way this suite touches export pricing. + ## Keeping the API contract honest `tests/contract/` checks this SDK's types against two pinned references: the diff --git a/package.json b/package.json index adcc482..5e49ec7 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,9 @@ "test": "vitest run", "test:watch": "vitest", "test:contract": "vitest run tests/contract", + "test:live": "vitest run --config vitest.live.config.ts", "test:package": "node scripts/verify-package.mjs", - "lint:package": "npx -y publint@latest --strict --pack npm && npx -y @arethetypeswrong/cli@latest --pack . --ignore-rules cjs-resolves-to-esm", + "lint:package": "npm run --silent build && npx -y publint@latest --strict --pack npm && npx -y @arethetypeswrong/cli@latest --pack . --ignore-rules cjs-resolves-to-esm", "spec:refresh": "curl -fsSL https://docs.tako.com/api-reference/openapi.yaml -o tests/contract/openapi.yaml", "prepublishOnly": "pnpm build" }, diff --git a/tests/live/options.live.test.ts b/tests/live/options.live.test.ts new file mode 100644 index 0000000..cbeddca --- /dev/null +++ b/tests/live/options.live.test.ts @@ -0,0 +1,212 @@ +/** + * Live checks against the real Tako API. + * + * Every other suite in this repo proves a request body is *legal* against a + * vendored snapshot of the OpenAPI document. None of them prove the API honors + * the option, or that the snapshot still matches reality. That gap is why the + * 2.x types rotted for two months while 18 tests stayed green. + * + * These tests close it from the other side: they send real requests and validate + * the responses with the same ajv validators the contract suite uses, so a + * response that stops matching the vendored spec fails here even though nothing + * in this repo changed. That makes this the upstream drift detector the parity + * tests cannot be — those read a pinned snapshot and only move when a human runs + * `pnpm spec:refresh`. + * + * Excluded from `pnpm test`. Run with `pnpm test:live` and a key, or let the + * `live` workflow run it on a schedule. Without `TAKO_API_KEY` every test skips + * rather than fails, so a contributor with no key sees no red. + * + * Two rules for anything added here: + * + * 1. Assert contract, never content. "A card came back" is stable; "the first + * card is Nvidia revenue" is a ranking change away from a false alarm. + * 2. Never trigger a billed export. `quoteOnly` prices one for free, and that is + * the only way this file touches export pricing. + */ +import { describe, expect, it } from "vitest"; +import { check } from "../contract/spec"; +import { buildContentsRequestBody, buildSearchRequestBody } from "../../src/request"; +import type { TakoContentsConfig, TakoRetrievalConfig } from "../../src/types"; + +const KEY = process.env.TAKO_API_KEY ?? process.env.TAKO_API_TOKEN; + +// Defaults to the same host the SDK does. Set TAKO_BASE_URL to point at another +// environment; no hostname other than the public default is committed here. +const BASE = (process.env.TAKO_BASE_URL ?? "https://tako.com").replace(/\/+$/, ""); + +const TIMEOUT = 60_000; + +async function post(path: string, body: unknown) { + const response = await fetch(`${BASE}${path}`, { + method: "POST", + headers: { "X-API-Key": KEY as string, "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + const text = await response.text(); + let json: Record | null = null; + try { + json = JSON.parse(text) as Record; + } catch { + // A non-JSON body is itself the failure; the assertion reports `text`. + } + return { status: response.status, json, text }; +} + +const search = (config: TakoRetrievalConfig, query = "nvidia revenue") => + post("/api/v3/search", buildSearchRequestBody(config, query)); + +const hostsOf = (json: Record | null) => + ((json?.web_results as { url: string }[] | undefined) ?? []).map((w) => { + try { + return new URL(w.url).hostname.replace(/^www\./, ""); + } catch { + return ""; + } + }); + +describe.skipIf(!KEY)("live: the request options reach a real API", () => { + it( + "a query-only body is accepted, and matches one that restates the old defaults", + async () => { + // This is why the check exists. 3.0.0 stopped sending effort, country_code + // and locale, on the evidence that the spec documents exactly the values + // the SDK used to hardcode. "The document says X is the default" and "the + // API applies X when the key is absent" are different claims, and only + // this one tests the second. + const bare = buildSearchRequestBody({}, "nvidia revenue"); + expect(Object.keys(bare)).toEqual(["query"]); + + const [a, b] = await Promise.all([ + search({}), + search({ effort: "fast", countryCode: "US", locale: "en-US" }), + ]); + + expect(a.status, a.text.slice(0, 300)).toBe(200); + expect(b.status, b.text.slice(0, 300)).toBe(200); + expect(Object.keys(a.json ?? {}).sort()).toEqual(Object.keys(b.json ?? {}).sort()); + expect((a.json?.web_results as unknown[] | undefined)?.length).toBe( + (b.json?.web_results as unknown[] | undefined)?.length, + ); + }, + TIMEOUT, + ); + + // A rejected option is an HTTP 400 with "Extra inputs are not permitted", + // which is how `deferDataRetrieval` failed for two months while typed as valid. + const accepted: [string, TakoRetrievalConfig][] = [ + ["location", { location: { latitude: 37.77, longitude: -122.42 } }], + ["data.contentFormat", { sources: { data: { includeContents: true, contentFormat: "json_records" } } }], + ["data.mode", { sources: { data: { includeContents: true, mode: "inline" } } }], + ["web.category", { sources: { web: { category: "news" } } }], + ["web.includeDomains", { sources: { web: { includeDomains: ["reuters.com"] } } }], + ["web.excludeDomains", { sources: { web: { excludeDomains: ["reddit.com"] } } }], + ["web.snippetMaxChars", { sources: { web: { snippetMaxChars: 300 } } }], + ["web.articleContentMaxChars", { sources: { web: { includeContents: true, articleContentMaxChars: 5000 } } }], + ["web.publishedAfter", { sources: { web: { publishedAfter: "2026-01-01" } } }], + ["web.publishedBefore", { sources: { web: { publishedBefore: "2026-12-31" } } }], + ]; + + it.each(accepted)("the API accepts %s", async (_name, config) => { + const r = await search(config); + expect(r.status, r.text.slice(0, 300)).toBe(200); + }, TIMEOUT); + + it( + "a live search response still validates against the vendored spec", + async () => { + // The drift detector. Nothing in this repo has to change for this to fail — + // it fails when the API stops matching the snapshot the other suites trust. + const r = await search({ sources: { data: { includeContents: true }, web: { count: 3 } } }); + expect(r.status).toBe(200); + expect(check("SearchResponse", r.json).errors).toEqual([]); + }, + TIMEOUT, + ); + + it( + "includeDomains and excludeDomains actually filter", + async () => { + const only = await search( + { sources: { web: { includeDomains: ["reuters.com"], count: 5 } } }, + "nvidia earnings", + ); + expect(only.status).toBe(200); + const kept = hostsOf(only.json); + // An empty list means the filter applied and nothing matched, which is a + // pass for "it filters" — a leak is the failure. + expect(kept.filter((h) => h && !h.endsWith("reuters.com"))).toEqual([]); + + const unfiltered = await search({ sources: { web: { count: 5 } } }, "nvidia earnings"); + const drop = hostsOf(unfiltered.json)[0]; + if (drop) { + const without = await search( + { sources: { web: { excludeDomains: [drop], count: 5 } } }, + "nvidia earnings", + ); + expect(without.status).toBe(200); + expect(hostsOf(without.json)).not.toContain(drop); + } + }, + TIMEOUT * 2, + ); + + it.each([ + ["json_records", "records"], + ["json_compact", "dataset"], + ] as const)( + "contentFormat %s is honored and populates %s", + async (format, field) => { + const r = await search({ + sources: { data: { includeContents: true, contentFormat: format, count: 1 } }, + }); + expect(r.status).toBe(200); + const content = (r.json?.cards as { content?: Record }[] | undefined)?.[0] + ?.content; + if (!content) return; // No inlined card to inspect; nothing to assert. + expect(content.content_format).toBe(format); + expect(content[field]).not.toBeNull(); + }, + TIMEOUT, + ); +}); + +describe.skipIf(!KEY)("live: contents pricing, quoted rather than bought", () => { + it( + "quoteOnly returns a price and no content, and maxRows clamps instead of failing", + async () => { + const seed = await search({}); + expect(seed.status).toBe(200); + const url = (seed.json?.cards as { webpage_url?: string }[] | undefined)?.[0]?.webpage_url; + if (!url) return; // No card to quote against. + + const quote = async (config: TakoContentsConfig) => { + const r = await post("/api/v1/contents", buildContentsRequestBody(url, config)); + expect(r.status, r.text.slice(0, 300)).toBe(200); + expect(check("ContentsResponse", r.json).errors).toEqual([]); + return (r.json?.contents as Record[] | undefined)?.[0]; + }; + + const small = await quote({ quoteOnly: true, maxRows: 20 }); + const large = await quote({ quoteOnly: true, maxRows: 2000 }); + if (!small || !large) return; + + // A quote carries pricing and withholds content. + expect(small.url ?? null).toBeNull(); + expect(small.data ?? null).toBeNull(); + expect(small.export_pricing).toBeTruthy(); + + // `cost` on a quote is the price the export would be, not a charge — which + // is only observable because it scales with maxRows. + expect(typeof small.cost).toBe("number"); + expect(large.cost as number).toBeGreaterThan(small.cost as number); + + // The documented behavior worth a live test: over the ceiling the API + // clamps and bills what it returns, so a caller trusting a 400 gets a + // short export, a charge, and no error. + const over = await quote({ quoteOnly: true, maxRows: 999_999 }); + if (over) expect(over.cost).toBe(large.cost); + }, + TIMEOUT * 3, + ); +}); diff --git a/vitest.config.ts b/vitest.config.ts index faa6d98..223c49c 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,8 +1,12 @@ -import { defineConfig } from "vitest/config"; +import { configDefaults, defineConfig } from "vitest/config"; export default defineConfig({ test: { environment: "node", include: ["tests/**/*.test.ts"], + // `tests/live/` makes real, billed API calls and needs a key. It must never + // run as part of `pnpm test`, which every contributor and CI job runs. + // `vitest.live.config.ts` is the only config that includes it. + exclude: [...configDefaults.exclude, "tests/live/**"], }, }); diff --git a/vitest.live.config.ts b/vitest.live.config.ts new file mode 100644 index 0000000..c59f068 --- /dev/null +++ b/vitest.live.config.ts @@ -0,0 +1,20 @@ +import { defineConfig } from "vitest/config"; + +/** + * The live suite only. Kept in its own config so `pnpm test` cannot pick these + * up: they need a real key and they cost money. + * + * Serial by design. The tests compare responses across requests, and Tako + * throttles per key, so a parallel run makes both the comparisons and the rate + * limit unpredictable. + */ +export default defineConfig({ + test: { + environment: "node", + include: ["tests/live/**/*.test.ts"], + testTimeout: 60_000, + hookTimeout: 60_000, + fileParallelism: false, + retry: 1, // one retry absorbs a transient 5xx without hiding a real failure + }, +});