diff --git a/.github/workflows/live.yml b/.github/workflows/live.yml new file mode 100644 index 0000000..a500fcd --- /dev/null +++ b/.github/workflows/live.yml @@ -0,0 +1,37 @@ +name: live + +# Never on pull_request. These tests spend money on every run, and a pull request +# from a fork cannot read secrets, so it would fail for everyone outside the org. +on: + workflow_dispatch: + schedule: + # Mondays, 13:00 UTC. The point of a schedule is to notice the API moving + # under a published version: the contract suite reads a pinned snapshot and + # only moves when somebody runs `pnpm spec:refresh`, so nothing else fails + # when Tako changes and this repo does not. + - cron: "0 13 * * 1" + +# One run at a time. Concurrent runs on the same key hit Tako's throttle and +# report a rate limit as a contract failure. +concurrency: + group: live-api + cancel-in-progress: false + +jobs: + live: + runs-on: ubuntu-latest + # Absent secret means an unrunnable job, which should not read as a failure. + if: ${{ github.event_name == 'workflow_dispatch' || vars.LIVE_TESTS_ENABLED == 'true' }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm test:live + env: + TAKO_API_KEY: ${{ secrets.TAKO_API_KEY }} + # Optional. Leave unset to test the same host the SDK defaults to. + TAKO_BASE_URL: ${{ vars.TAKO_BASE_URL }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 748b88f..4c6107b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,6 +10,7 @@ pnpm typecheck # tsc over src + tests + examples pnpm build # tsup → dist/ pnpm lint:package # publint + are-the-types-wrong, against the packed tarball pnpm test:package # install the tarball in a scratch project and use it +pnpm test:live # real API calls — needs TAKO_API_KEY, costs money pnpm spec:refresh # re-vendor tests/contract/openapi.yaml from docs.tako.com ``` @@ -48,6 +49,41 @@ every `.d.ts` under `node_modules`, and the `ai` package's own tree reports missing `@types/node` and `@types/json-schema` — another package's noise, loud enough to hide a real failure here. +## Checking the API itself (`tests/live/`) + +Every other suite proves a request body is **legal** against a vendored snapshot of +the OpenAPI document. None of them prove the API **honors** the option, or that the +snapshot still matches reality — the gap that let the 2.x types rot for two months +while every test stayed green. + +`tests/live/` closes it from the other side. It sends real requests and validates +the responses with the same ajv validators the contract suite uses, so **a response +that stops matching the vendored spec fails even though nothing in this repo +changed.** That makes it the upstream drift detector the parity tests cannot be: +those read a pinned snapshot and only move when someone runs `pnpm spec:refresh`. + +```bash +TAKO_API_KEY=... pnpm test:live +TAKO_API_KEY=... TAKO_BASE_URL=https://some-other-host pnpm test:live # optional +``` + +- **Excluded from `pnpm test`** by `vitest.config.ts`, and only included by + `vitest.live.config.ts`. It costs money, so it must never run by accident. +- **Skips without a key** rather than failing, so a contributor with no key sees + no red. +- **Runs on a schedule** (`.github/workflows/live.yml`, Mondays 13:00 UTC) plus + manual dispatch. Never on `pull_request`: forks cannot read secrets, so it + would fail for every outside contributor. +- Serial, with one retry, because the tests compare responses across requests and + Tako throttles per key. + +Two rules for anything you add there: + +1. **Assert contract, never content.** "A card came back" is stable. "The first + card is Nvidia revenue" is one ranking change from a false alarm. +2. **Never trigger a billed export.** `quoteOnly` prices one for free, and that is + the only way this suite touches export pricing. + ## Keeping the API contract honest `tests/contract/` checks this SDK's types against two pinned references: the diff --git a/package.json b/package.json index adcc482..5e49ec7 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,9 @@ "test": "vitest run", "test:watch": "vitest", "test:contract": "vitest run tests/contract", + "test:live": "vitest run --config vitest.live.config.ts", "test:package": "node scripts/verify-package.mjs", - "lint:package": "npx -y publint@latest --strict --pack npm && npx -y @arethetypeswrong/cli@latest --pack . --ignore-rules cjs-resolves-to-esm", + "lint:package": "npm run --silent build && npx -y publint@latest --strict --pack npm && npx -y @arethetypeswrong/cli@latest --pack . --ignore-rules cjs-resolves-to-esm", "spec:refresh": "curl -fsSL https://docs.tako.com/api-reference/openapi.yaml -o tests/contract/openapi.yaml", "prepublishOnly": "pnpm build" }, diff --git a/tests/live/options.live.test.ts b/tests/live/options.live.test.ts new file mode 100644 index 0000000..cbeddca --- /dev/null +++ b/tests/live/options.live.test.ts @@ -0,0 +1,212 @@ +/** + * Live checks against the real Tako API. + * + * Every other suite in this repo proves a request body is *legal* against a + * vendored snapshot of the OpenAPI document. None of them prove the API honors + * the option, or that the snapshot still matches reality. That gap is why the + * 2.x types rotted for two months while 18 tests stayed green. + * + * These tests close it from the other side: they send real requests and validate + * the responses with the same ajv validators the contract suite uses, so a + * response that stops matching the vendored spec fails here even though nothing + * in this repo changed. That makes this the upstream drift detector the parity + * tests cannot be — those read a pinned snapshot and only move when a human runs + * `pnpm spec:refresh`. + * + * Excluded from `pnpm test`. Run with `pnpm test:live` and a key, or let the + * `live` workflow run it on a schedule. Without `TAKO_API_KEY` every test skips + * rather than fails, so a contributor with no key sees no red. + * + * Two rules for anything added here: + * + * 1. Assert contract, never content. "A card came back" is stable; "the first + * card is Nvidia revenue" is a ranking change away from a false alarm. + * 2. Never trigger a billed export. `quoteOnly` prices one for free, and that is + * the only way this file touches export pricing. + */ +import { describe, expect, it } from "vitest"; +import { check } from "../contract/spec"; +import { buildContentsRequestBody, buildSearchRequestBody } from "../../src/request"; +import type { TakoContentsConfig, TakoRetrievalConfig } from "../../src/types"; + +const KEY = process.env.TAKO_API_KEY ?? process.env.TAKO_API_TOKEN; + +// Defaults to the same host the SDK does. Set TAKO_BASE_URL to point at another +// environment; no hostname other than the public default is committed here. +const BASE = (process.env.TAKO_BASE_URL ?? "https://tako.com").replace(/\/+$/, ""); + +const TIMEOUT = 60_000; + +async function post(path: string, body: unknown) { + const response = await fetch(`${BASE}${path}`, { + method: "POST", + headers: { "X-API-Key": KEY as string, "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + const text = await response.text(); + let json: Record | null = null; + try { + json = JSON.parse(text) as Record; + } catch { + // A non-JSON body is itself the failure; the assertion reports `text`. + } + return { status: response.status, json, text }; +} + +const search = (config: TakoRetrievalConfig, query = "nvidia revenue") => + post("/api/v3/search", buildSearchRequestBody(config, query)); + +const hostsOf = (json: Record | null) => + ((json?.web_results as { url: string }[] | undefined) ?? []).map((w) => { + try { + return new URL(w.url).hostname.replace(/^www\./, ""); + } catch { + return ""; + } + }); + +describe.skipIf(!KEY)("live: the request options reach a real API", () => { + it( + "a query-only body is accepted, and matches one that restates the old defaults", + async () => { + // This is why the check exists. 3.0.0 stopped sending effort, country_code + // and locale, on the evidence that the spec documents exactly the values + // the SDK used to hardcode. "The document says X is the default" and "the + // API applies X when the key is absent" are different claims, and only + // this one tests the second. + const bare = buildSearchRequestBody({}, "nvidia revenue"); + expect(Object.keys(bare)).toEqual(["query"]); + + const [a, b] = await Promise.all([ + search({}), + search({ effort: "fast", countryCode: "US", locale: "en-US" }), + ]); + + expect(a.status, a.text.slice(0, 300)).toBe(200); + expect(b.status, b.text.slice(0, 300)).toBe(200); + expect(Object.keys(a.json ?? {}).sort()).toEqual(Object.keys(b.json ?? {}).sort()); + expect((a.json?.web_results as unknown[] | undefined)?.length).toBe( + (b.json?.web_results as unknown[] | undefined)?.length, + ); + }, + TIMEOUT, + ); + + // A rejected option is an HTTP 400 with "Extra inputs are not permitted", + // which is how `deferDataRetrieval` failed for two months while typed as valid. + const accepted: [string, TakoRetrievalConfig][] = [ + ["location", { location: { latitude: 37.77, longitude: -122.42 } }], + ["data.contentFormat", { sources: { data: { includeContents: true, contentFormat: "json_records" } } }], + ["data.mode", { sources: { data: { includeContents: true, mode: "inline" } } }], + ["web.category", { sources: { web: { category: "news" } } }], + ["web.includeDomains", { sources: { web: { includeDomains: ["reuters.com"] } } }], + ["web.excludeDomains", { sources: { web: { excludeDomains: ["reddit.com"] } } }], + ["web.snippetMaxChars", { sources: { web: { snippetMaxChars: 300 } } }], + ["web.articleContentMaxChars", { sources: { web: { includeContents: true, articleContentMaxChars: 5000 } } }], + ["web.publishedAfter", { sources: { web: { publishedAfter: "2026-01-01" } } }], + ["web.publishedBefore", { sources: { web: { publishedBefore: "2026-12-31" } } }], + ]; + + it.each(accepted)("the API accepts %s", async (_name, config) => { + const r = await search(config); + expect(r.status, r.text.slice(0, 300)).toBe(200); + }, TIMEOUT); + + it( + "a live search response still validates against the vendored spec", + async () => { + // The drift detector. Nothing in this repo has to change for this to fail — + // it fails when the API stops matching the snapshot the other suites trust. + const r = await search({ sources: { data: { includeContents: true }, web: { count: 3 } } }); + expect(r.status).toBe(200); + expect(check("SearchResponse", r.json).errors).toEqual([]); + }, + TIMEOUT, + ); + + it( + "includeDomains and excludeDomains actually filter", + async () => { + const only = await search( + { sources: { web: { includeDomains: ["reuters.com"], count: 5 } } }, + "nvidia earnings", + ); + expect(only.status).toBe(200); + const kept = hostsOf(only.json); + // An empty list means the filter applied and nothing matched, which is a + // pass for "it filters" — a leak is the failure. + expect(kept.filter((h) => h && !h.endsWith("reuters.com"))).toEqual([]); + + const unfiltered = await search({ sources: { web: { count: 5 } } }, "nvidia earnings"); + const drop = hostsOf(unfiltered.json)[0]; + if (drop) { + const without = await search( + { sources: { web: { excludeDomains: [drop], count: 5 } } }, + "nvidia earnings", + ); + expect(without.status).toBe(200); + expect(hostsOf(without.json)).not.toContain(drop); + } + }, + TIMEOUT * 2, + ); + + it.each([ + ["json_records", "records"], + ["json_compact", "dataset"], + ] as const)( + "contentFormat %s is honored and populates %s", + async (format, field) => { + const r = await search({ + sources: { data: { includeContents: true, contentFormat: format, count: 1 } }, + }); + expect(r.status).toBe(200); + const content = (r.json?.cards as { content?: Record }[] | undefined)?.[0] + ?.content; + if (!content) return; // No inlined card to inspect; nothing to assert. + expect(content.content_format).toBe(format); + expect(content[field]).not.toBeNull(); + }, + TIMEOUT, + ); +}); + +describe.skipIf(!KEY)("live: contents pricing, quoted rather than bought", () => { + it( + "quoteOnly returns a price and no content, and maxRows clamps instead of failing", + async () => { + const seed = await search({}); + expect(seed.status).toBe(200); + const url = (seed.json?.cards as { webpage_url?: string }[] | undefined)?.[0]?.webpage_url; + if (!url) return; // No card to quote against. + + const quote = async (config: TakoContentsConfig) => { + const r = await post("/api/v1/contents", buildContentsRequestBody(url, config)); + expect(r.status, r.text.slice(0, 300)).toBe(200); + expect(check("ContentsResponse", r.json).errors).toEqual([]); + return (r.json?.contents as Record[] | undefined)?.[0]; + }; + + const small = await quote({ quoteOnly: true, maxRows: 20 }); + const large = await quote({ quoteOnly: true, maxRows: 2000 }); + if (!small || !large) return; + + // A quote carries pricing and withholds content. + expect(small.url ?? null).toBeNull(); + expect(small.data ?? null).toBeNull(); + expect(small.export_pricing).toBeTruthy(); + + // `cost` on a quote is the price the export would be, not a charge — which + // is only observable because it scales with maxRows. + expect(typeof small.cost).toBe("number"); + expect(large.cost as number).toBeGreaterThan(small.cost as number); + + // The documented behavior worth a live test: over the ceiling the API + // clamps and bills what it returns, so a caller trusting a 400 gets a + // short export, a charge, and no error. + const over = await quote({ quoteOnly: true, maxRows: 999_999 }); + if (over) expect(over.cost).toBe(large.cost); + }, + TIMEOUT * 3, + ); +}); diff --git a/vitest.config.ts b/vitest.config.ts index faa6d98..223c49c 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,8 +1,12 @@ -import { defineConfig } from "vitest/config"; +import { configDefaults, defineConfig } from "vitest/config"; export default defineConfig({ test: { environment: "node", include: ["tests/**/*.test.ts"], + // `tests/live/` makes real, billed API calls and needs a key. It must never + // run as part of `pnpm test`, which every contributor and CI job runs. + // `vitest.live.config.ts` is the only config that includes it. + exclude: [...configDefaults.exclude, "tests/live/**"], }, }); diff --git a/vitest.live.config.ts b/vitest.live.config.ts new file mode 100644 index 0000000..c59f068 --- /dev/null +++ b/vitest.live.config.ts @@ -0,0 +1,20 @@ +import { defineConfig } from "vitest/config"; + +/** + * The live suite only. Kept in its own config so `pnpm test` cannot pick these + * up: they need a real key and they cost money. + * + * Serial by design. The tests compare responses across requests, and Tako + * throttles per key, so a parallel run makes both the comparisons and the rate + * limit unpredictable. + */ +export default defineConfig({ + test: { + environment: "node", + include: ["tests/live/**/*.test.ts"], + testTimeout: 60_000, + hookTimeout: 60_000, + fileParallelism: false, + retry: 1, // one retry absorbs a transient 5xx without hiding a real failure + }, +});