diff --git a/.env.example b/.env.example index 17ada6e04..bb617c483 100644 --- a/.env.example +++ b/.env.example @@ -10,3 +10,8 @@ POSTHOG_PERSONAL_API_KEY=phx_... # POSTHOG_REGION is optional, defaults to 'us'. Can also be passed via --region flag or workflow input. # POSTHOG_REGION=us POSTHOG_WIZARD_PROJECT_ID=123 + +# AI gateway token for --e2e runs. Leave unset locally: the run mints one per program, region and project through the +# wizard's `pnpm mint-gateway-token` (one browser login, cached up to 24h in ~/.config/posthog). +# Set it to a file holding an already-issued token (as CI does) to skip minting. +# WIZARD_CI_GATEWAY_TOKEN_FILE= diff --git a/package.json b/package.json index ed8bff2ab..51144923d 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "type": "module", "scripts": { "evaluate": "tsx services/pr-evaluator/index.ts", + "gateway-token": "tsx services/gateway-token/index.ts", "wizard-ci": "tsx services/wizard-ci/index.ts", "wizard-ci-snapshots": "tsx services/wizard-ci/snapshots.ts", "wizard-ci-snapshots-review": "tsx services/wizard-ci/snapshots-review.ts", @@ -16,7 +17,8 @@ "test:mcp-stub": "tsx --test services/mcp-stub/mcp-stub.test.ts", "test:warehouse-checks": "tsx --test services/wizard-ci/warehouse-checks.test.ts", "test:feature-flag-checks": "tsx --test services/wizard-ci/feature-flag-checks.test.ts", - "test:source-app": "tsx --test services/wizard-ci/source-app.test.ts" + "test:source-app": "tsx --test services/wizard-ci/source-app.test.ts", + "test:gateway-token": "tsx --test services/gateway-token/gateway-token.test.ts" }, "dependencies": { "@anthropic-ai/claude-agent-sdk": "0.2.73", diff --git a/services/gateway-token/gateway-token.test.ts b/services/gateway-token/gateway-token.test.ts new file mode 100644 index 000000000..204dfadb0 --- /dev/null +++ b/services/gateway-token/gateway-token.test.ts @@ -0,0 +1,273 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { afterEach, beforeEach, describe, it } from "node:test"; + +import { + defaultGatewayTokenFile, + ensureGatewayToken, + type GatewayToken, + type MintRequest, + type SavedGatewayToken, +} from "./index.js"; + +const NOW_MS = 1_800_000_000_000; +const GATEWAY_URL = "https://ai-gateway.us.posthog.com"; +const SAVED_TOKEN = "phe_saved"; +const SAVED_TOKEN_SHA256 = createHash("sha256").update(SAVED_TOKEN).digest("hex"); + +function saveToken(tokenFile: string, saved: SavedGatewayToken): void { + mkdirSync(dirname(tokenFile), { recursive: true }); + writeFileSync(tokenFile, SAVED_TOKEN); + writeFileSync(`${tokenFile}.json`, JSON.stringify({ tokenSha256: SAVED_TOKEN_SHA256, ...saved })); +} + +function assertSnapshotOf(token: GatewayToken, sourceTokenFile: string): void { + assert.equal(token.sourceTokenFile, sourceTokenFile); + assert.notEqual(token.tokenFile, sourceTokenFile); + assert.deepEqual(readFileSync(token.tokenFile), readFileSync(sourceTokenFile)); + assert.equal(statSync(token.tokenFile).mode & 0o777, 0o600); +} + +describe("ensureGatewayToken", () => { + let workDir: string; + let tokenFile: string; + let mintRequests: MintRequest[]; + let issuedTokens: GatewayToken[]; + + const ensureAndTrack = async (tokenOptions: Parameters[0]) => { + const token = await ensureGatewayToken(tokenOptions); + issuedTokens.push(token); + return token; + }; + + const recordingMint = async (request: MintRequest) => { + mintRequests.push(request); + saveToken(request.tokenFile, { + program: request.program, + projectId: Number(request.projectId), + gatewayUrl: GATEWAY_URL, + refreshAtMs: NOW_MS + 60_000, + }); + }; + + const options = (overrides: Partial[0]> = {}) => ({ + program: "feature-flags", + projectId: "483112", + tokenFile, + wizardPath: "/wizard", + environment: {}, + now: () => NOW_MS, + mintToken: recordingMint, + ...overrides, + }); + + beforeEach(() => { + workDir = mkdtempSync(join(tmpdir(), "gateway-token-test-")); + tokenFile = join(workDir, "posthog", "wizard-gateway-token"); + mintRequests = []; + issuedTokens = []; + }); + + afterEach(() => { + for (const token of issuedTokens) token.dispose(); + rmSync(workDir, { recursive: true, force: true }); + }); + + it("mints through the wizard login when no token is saved", async () => { + const token = await ensureAndTrack(options()); + + assert.deepEqual(mintRequests, [ + { program: "feature-flags", projectId: "483112", tokenFile, wizardPath: "/wizard" }, + ]); + assertSnapshotOf(token, tokenFile); + assert.equal(token.gatewayUrl, GATEWAY_URL); + }); + + it("hands out a snapshot of a reused token that later writes to the cache file do not change", async () => { + saveToken(tokenFile, { program: "feature-flags", projectId: 483112, gatewayUrl: GATEWAY_URL, refreshAtMs: NOW_MS + 1 }); + + const token = await ensureAndTrack(options()); + writeFileSync(tokenFile, "phe_from_another_run"); + + assert.equal(readFileSync(token.tokenFile, "utf8"), SAVED_TOKEN); + }); + + it("removes the token snapshot on dispose", async () => { + const token = await ensureAndTrack(options()); + + token.dispose(); + + assert.equal(existsSync(token.tokenFile), false); + assert.equal(existsSync(dirname(token.tokenFile)), false); + assert.equal(existsSync(tokenFile), true); + }); + + it("reuses a saved token for the same program and project before its refresh time", async () => { + saveToken(tokenFile, { program: "feature-flags", projectId: 483112, gatewayUrl: GATEWAY_URL, refreshAtMs: NOW_MS + 1 }); + + await ensureAndTrack(options()); + + assert.equal(mintRequests.length, 0); + }); + + it("re-mints once the saved token is past its refresh time", async () => { + saveToken(tokenFile, { program: "feature-flags", projectId: 483112, gatewayUrl: GATEWAY_URL, refreshAtMs: NOW_MS }); + + await ensureAndTrack(options()); + + assert.equal(mintRequests.length, 1); + }); + + it("re-mints when the saved token was minted for another program", async () => { + saveToken(tokenFile, { program: "error-tracking", projectId: 483112, gatewayUrl: GATEWAY_URL, refreshAtMs: NOW_MS + 60_000 }); + + await ensureAndTrack(options()); + + assert.equal(mintRequests[0]?.program, "feature-flags"); + }); + + it("re-mints when the saved token belongs to another project", async () => { + saveToken(tokenFile, { program: "feature-flags", projectId: 1, gatewayUrl: GATEWAY_URL, refreshAtMs: NOW_MS + 60_000 }); + + await ensureAndTrack(options()); + + assert.equal(mintRequests.length, 1); + }); + + const withHomeDirectory = async (run: () => Promise) => { + const realHome = process.env.HOME; + process.env.HOME = workDir; + try { + await run(); + } finally { + process.env.HOME = realHome; + } + }; + + it("uses an explicitly given token file without a sidecar as-is and never mints", async () => { + mkdirSync(dirname(tokenFile), { recursive: true }); + writeFileSync(tokenFile, "phe_issued_by_ci"); + + const token = await ensureAndTrack(options()); + + assert.equal(mintRequests.length, 0); + assertSnapshotOf(token, tokenFile); + assert.equal(token.gatewayUrl, undefined); + }); + + it("mints into a per-program, per-region, per-project file under the home directory when no token file is given", async () => { + await withHomeDirectory(async () => { + const token = await ensureAndTrack( + options({ tokenFile: undefined, program: "posthog-integration", region: "us" }), + ); + + const expectedFile = join(workDir, ".config", "posthog", "wizard-gateway-token-posthog-integration-us-483112"); + assert.equal(defaultGatewayTokenFile("posthog-integration", "us", "483112"), expectedFile); + assert.equal(mintRequests[0]?.tokenFile, expectedFile); + assertSnapshotOf(token, expectedFile); + }); + }); + + it("re-mints a sidecarless token in the managed default file", async () => { + await withHomeDirectory(async () => { + const managedFile = defaultGatewayTokenFile("feature-flags", "us", "483112"); + mkdirSync(dirname(managedFile), { recursive: true }); + writeFileSync(managedFile, "phe_interrupted_mint"); + + const token = await ensureAndTrack(options({ tokenFile: undefined })); + + assert.equal(mintRequests.length, 1); + assertSnapshotOf(token, managedFile); + assert.equal(token.gatewayUrl, GATEWAY_URL); + }); + }); + + it("re-mints when the saved token's gateway belongs to another region", async () => { + saveToken(tokenFile, { + program: "feature-flags", + projectId: 483112, + gatewayUrl: "https://ai-gateway.eu.posthog.com", + refreshAtMs: NOW_MS + 60_000, + }); + + const token = await ensureAndTrack(options({ region: "us" })); + + assert.equal(mintRequests.length, 1); + assert.equal(token.gatewayUrl, GATEWAY_URL); + }); + + it("re-mints when the saved gateway does not match an explicit gateway override", async () => { + saveToken(tokenFile, { + program: "feature-flags", + projectId: 483112, + gatewayUrl: "http://localhost:8765", + refreshAtMs: NOW_MS + 60_000, + }); + + const token = await ensureAndTrack(options({ environment: { WIZARD_CI_GATEWAY_URL: `${GATEWAY_URL}/v1` } })); + + assert.equal(mintRequests.length, 1); + assert.equal(token.gatewayUrl, GATEWAY_URL); + }); + + it("reuses a saved token whose gateway matches the origin of an explicit gateway override", async () => { + saveToken(tokenFile, { program: "feature-flags", projectId: 483112, gatewayUrl: GATEWAY_URL, refreshAtMs: NOW_MS + 1 }); + + await ensureAndTrack(options({ environment: { WIZARD_CI_GATEWAY_URL: `${GATEWAY_URL}/v1` } })); + + assert.equal(mintRequests.length, 0); + }); + + it("re-mints when the sidecar hash does not match the token file", async () => { + saveToken(tokenFile, { + program: "feature-flags", + projectId: 483112, + gatewayUrl: GATEWAY_URL, + refreshAtMs: NOW_MS + 60_000, + tokenSha256: createHash("sha256").update("phe_previous").digest("hex"), + }); + + await ensureAndTrack(options()); + + assert.equal(mintRequests.length, 1); + }); + + it("re-mints when the sidecar has no token hash", async () => { + saveToken(tokenFile, { + program: "feature-flags", + projectId: 483112, + gatewayUrl: GATEWAY_URL, + refreshAtMs: NOW_MS + 60_000, + tokenSha256: undefined, + }); + + await ensureAndTrack(options()); + + assert.equal(mintRequests.length, 1); + }); + + it("fails without minting in CI when no usable token exists", async () => { + await assert.rejects( + ensureGatewayToken(options({ environment: { CI: "true" } })), + /WIZARD_CI_GATEWAY_TOKEN_FILE/, + ); + assert.equal(mintRequests.length, 0); + }); + + it("names the missing wizard mint script instead of spawning it", async () => { + await assert.rejects( + ensureGatewayToken(options({ mintToken: undefined, wizardPath: workDir })), + /has no scripts\/mint-gateway-token\.no-jest\.ts/, + ); + }); + + it("fails when the mint leaves no usable token behind", async () => { + await assert.rejects( + ensureGatewayToken(options({ mintToken: async () => undefined })), + /did not leave a usable token for feature-flags/, + ); + }); +}); diff --git a/services/gateway-token/index.ts b/services/gateway-token/index.ts new file mode 100644 index 000000000..a97ed6e1e --- /dev/null +++ b/services/gateway-token/index.ts @@ -0,0 +1,213 @@ +import "dotenv/config"; +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { homedir, tmpdir } from "node:os"; +import { join } from "node:path"; + +const WIZARD_MINT_SCRIPT = join("scripts", "mint-gateway-token.no-jest.ts"); +const WORKBENCH = join(import.meta.dirname, "..", ".."); + +export function resolveWizardRepo(): string { + const configuredPath = process.env.WIZARD_PATH?.replace(/^~/, process.env.HOME || ""); + if (configuredPath) return configuredPath; + for (const name of ["wizard-e2e", "wizard"]) { + const siblingPath = join(WORKBENCH, "..", name); + if (existsSync(siblingPath)) return siblingPath; + } + return `${process.env.HOME}/development/wizard`; +} + +export function defaultGatewayTokenFile(program: string, region: string, projectId: string): string { + return join(homedir(), ".config", "posthog", `wizard-gateway-token-${program}-${region}-${projectId}`); +} + +export interface SavedGatewayToken { + program: string; + projectId: number; + gatewayUrl: string; + refreshAtMs: number; + tokenSha256?: string; +} + +export interface GatewayToken { + tokenFile: string; + sourceTokenFile: string; + gatewayUrl?: string; + dispose: () => void; +} + +export interface GatewayTokenOptions { + program: string; + projectId: string; + region?: string; + tokenFile?: string; + wizardPath?: string; + refresh?: boolean; + environment?: NodeJS.ProcessEnv; + now?: () => number; + mintToken?: (request: MintRequest) => Promise; +} + +export interface MintRequest { + program: string; + projectId: string; + tokenFile: string; + wizardPath: string; +} + +function isGatewayForRegion(gatewayUrl: string, region: string, environment: NodeJS.ProcessEnv): boolean { + if (!URL.canParse(gatewayUrl)) return false; + const sidecarOrigin = new URL(gatewayUrl).origin; + const overrideUrl = environment.WIZARD_CI_GATEWAY_URL; + if (!overrideUrl) return sidecarOrigin === `https://ai-gateway.${region}.posthog.com`; + if (!URL.canParse(overrideUrl)) return false; + return sidecarOrigin === new URL(overrideUrl).origin; +} + +function isSidecarBoundToToken(tokenBytes: Buffer, saved: SavedGatewayToken): boolean { + if (!saved.tokenSha256) return false; + return createHash("sha256").update(tokenBytes).digest("hex") === saved.tokenSha256; +} + +function readPreIssuedTokenBytes(tokenFile: string): Buffer | undefined { + if (!existsSync(tokenFile) || existsSync(`${tokenFile}.json`)) return undefined; + const tokenBytes = readFileSync(tokenFile); + if (tokenBytes.toString("utf8").trim() === "") return undefined; + return tokenBytes; +} + +function snapshotGatewayToken( + tokenBytes: Buffer, + sourceTokenFile: string, + gatewayUrl?: string, +): GatewayToken { + const snapshotDirectory = mkdtempSync(join(tmpdir(), "wizard-gateway-token-")); + const dispose = () => rmSync(snapshotDirectory, { recursive: true, force: true }); + const snapshotFile = join(snapshotDirectory, "token"); + try { + writeFileSync(snapshotFile, tokenBytes, { mode: 0o600, flag: "wx" }); + } catch (error) { + dispose(); + throw error; + } + return { tokenFile: snapshotFile, sourceTokenFile, gatewayUrl, dispose }; +} + +export function readSavedGatewayToken(tokenFile: string): SavedGatewayToken | undefined { + const metadataFile = `${tokenFile}.json`; + if (!existsSync(tokenFile) || !existsSync(metadataFile)) return undefined; + if (!readFileSync(tokenFile, "utf8").trim()) return undefined; + try { + return JSON.parse(readFileSync(metadataFile, "utf8")) as SavedGatewayToken; + } catch { + return undefined; + } +} + +export function isSavedTokenUsable( + saved: SavedGatewayToken | undefined, + program: string, + projectId: string, + nowMs: number, +): saved is SavedGatewayToken { + if (!saved) return false; + if (saved.program !== program) return false; + if (String(saved.projectId) !== projectId) return false; + return nowMs < saved.refreshAtMs; +} + +function mintWithWizardLogin(request: MintRequest): Promise { + if (!existsSync(join(request.wizardPath, WIZARD_MINT_SCRIPT))) { + return Promise.reject( + new Error( + `${request.wizardPath} has no ${WIZARD_MINT_SCRIPT}; update the wizard checkout, or set WIZARD_CI_GATEWAY_TOKEN_FILE to a file holding an issued token`, + ), + ); + } + const tsx = join(request.wizardPath, "node_modules", ".bin", "tsx"); + const child = spawn(tsx, [WIZARD_MINT_SCRIPT], { + cwd: request.wizardPath, + stdio: "inherit", + env: { + ...process.env, + PROGRAM: request.program, + PROJECT_ID: request.projectId, + TOKEN_FILE: request.tokenFile, + }, + }); + return new Promise((resolve, reject) => { + child.on("error", reject); + child.on("close", (code) => + code === 0 ? resolve() : reject(new Error(`gateway token mint exited with code ${code}`)), + ); + }); +} + +export async function ensureGatewayToken(options: GatewayTokenOptions): Promise { + const region = options.region || "us"; + const environment = options.environment ?? process.env; + const isExplicitTokenFile = Boolean(options.tokenFile); + const tokenFile = options.tokenFile || defaultGatewayTokenFile(options.program, region, options.projectId); + const preIssuedTokenBytes = isExplicitTokenFile ? readPreIssuedTokenBytes(tokenFile) : undefined; + if (preIssuedTokenBytes) return snapshotGatewayToken(preIssuedTokenBytes, tokenFile); + const nowMs = (options.now ?? Date.now)(); + const snapshotIfUsableForThisRun = (saved: SavedGatewayToken | undefined): GatewayToken | undefined => { + if (!isSavedTokenUsable(saved, options.program, options.projectId, nowMs)) return undefined; + if (!isGatewayForRegion(saved.gatewayUrl, region, environment)) return undefined; + const tokenBytes = readFileSync(tokenFile); + if (!isSidecarBoundToToken(tokenBytes, saved)) return undefined; + return snapshotGatewayToken(tokenBytes, tokenFile, saved.gatewayUrl); + }; + const cachedToken = options.refresh ? undefined : snapshotIfUsableForThisRun(readSavedGatewayToken(tokenFile)); + if (cachedToken) return cachedToken; + if (environment.CI) { + throw new Error( + `no usable gateway token for ${options.program} in ${tokenFile}; in CI set WIZARD_CI_GATEWAY_TOKEN_FILE to a file holding an issued token`, + ); + } + + const mintToken = options.mintToken ?? mintWithWizardLogin; + await mintToken({ + program: options.program, + projectId: options.projectId, + tokenFile, + wizardPath: options.wizardPath ?? resolveWizardRepo(), + }); + + const mintedToken = snapshotIfUsableForThisRun(readSavedGatewayToken(tokenFile)); + if (!mintedToken) { + throw new Error(`the mint did not leave a usable token for ${options.program} in ${tokenFile}`); + } + return mintedToken; +} + +function readFlag(name: string): string | undefined { + const index = process.argv.indexOf(`--${name}`); + return index === -1 ? undefined : process.argv[index + 1]; +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const projectId = readFlag("project-id") ?? process.env.POSTHOG_WIZARD_PROJECT_ID; + const program = readFlag("program") ?? "posthog-integration"; + if (!projectId) { + console.error("✖ project id required: --project-id or POSTHOG_WIZARD_PROJECT_ID."); + process.exit(2); + } + ensureGatewayToken({ + program, + projectId, + region: process.env.POSTHOG_REGION || "us", + tokenFile: process.env.WIZARD_CI_GATEWAY_TOKEN_FILE, + wizardPath: resolveWizardRepo(), + refresh: process.argv.includes("--refresh"), + }) + .then(({ sourceTokenFile, dispose }) => { + dispose(); + console.log(`✓ gateway token for ${program} ready: ${sourceTokenFile}`); + }) + .catch((error: Error) => { + console.error(`✖ ${error.message}`); + process.exit(1); + }); +} diff --git a/services/wizard-ci/e2e.ts b/services/wizard-ci/e2e.ts index da5e29677..570ab8823 100644 --- a/services/wizard-ci/e2e.ts +++ b/services/wizard-ci/e2e.ts @@ -22,6 +22,7 @@ import { readdirSync, statSync, } from "fs"; +import { ensureGatewayToken, resolveWizardRepo, type GatewayToken } from "../gateway-token/index.js"; import { loadFixtures } from "../mcp-stub/fixtures.js"; import { startMcpStub, type McpStub } from "../mcp-stub/index.js"; import { readJournal } from "../mcp-stub/journal.js"; @@ -124,17 +125,6 @@ export interface E2eOptions { triggerId?: string; } -function wizardRepo(): string { - const p = process.env.WIZARD_PATH?.replace(/^~/, process.env.HOME || ""); - if (p) return p; - // Default to a sibling wizard checkout next to the workbench. - for (const name of ["wizard-e2e", "wizard"]) { - const sibling = join(WORKBENCH, "..", name); - if (existsSync(sibling)) return sibling; - } - return `${process.env.HOME}/development/wizard`; -} - /** Where a run drops its real-TUI snapshots — shared with the snapshots flow. */ export function snapsDirFor(app: string): string { return `/tmp/wizard-e2e-${basename(app)}-snaps`; @@ -258,6 +248,45 @@ export async function runE2e(opts: E2eOptions): Promise { return 2; } + const program = opts.program ?? "posthog-integration"; + let gatewayToken: GatewayToken; + try { + gatewayToken = await ensureGatewayToken({ + program, + projectId, + region, + tokenFile: process.env.WIZARD_CI_GATEWAY_TOKEN_FILE, + wizardPath: resolveWizardRepo(), + }); + } catch (error) { + console.error(`✖ gateway token: ${(error as Error).message}`); + return 2; + } + + try { + return await runE2eWithGatewayToken({ opts, app, region, projectId, apiKey, program, gatewayToken }); + } finally { + gatewayToken.dispose(); + } +} + +async function runE2eWithGatewayToken({ + opts, + app, + region, + projectId, + apiKey, + program, + gatewayToken, +}: { + opts: E2eOptions; + app: string; + region: string; + projectId: string; + apiKey: string; + program: string; + gatewayToken: GatewayToken; +}): Promise { // A scenario may run against a sibling app's source tree (`sourceApp`), so a // run variation gets its own matrix leg without a second copy of the fixture. const expect = loadExpect(APPS_DIR, app); @@ -293,7 +322,7 @@ export async function runE2e(opts: E2eOptions): Promise { mkdirSync(snapsDir, { recursive: true }); rmSync(resultJson, { force: true }); - const repo = wizardRepo(); + const repo = resolveWizardRepo(); const harness = join(repo, "scripts", "tui-snapshots.no-jest.ts"); if (!existsSync(harness)) { console.error(`✖ wizard e2e harness not found: ${harness}\n Set WIZARD_PATH to the wizard repo.`); @@ -310,14 +339,15 @@ export async function runE2e(opts: E2eOptions): Promise { for (const k of Object.keys(childEnv)) if (STRIP_HOST_AUTH.test(k)) delete childEnv[k]; childEnv.POSTHOG_PERSONAL_API_KEY = apiKey; + childEnv.WIZARD_CI_GATEWAY_TOKEN_FILE = gatewayToken.tokenFile; + if (gatewayToken.gatewayUrl && !childEnv.WIZARD_CI_GATEWAY_URL) childEnv.WIZARD_CI_GATEWAY_URL = gatewayToken.gatewayUrl; childEnv.APP_DIR = appDir; childEnv.PROJECT_ID = projectId; childEnv.POSTHOG_REGION = region; childEnv.SNAP_OUT = snapsDir; childEnv.E2E_RESULT_JSON = resultJson; childEnv.E2E_KEEP_SKILLS = opts.keepSkills ? "true" : "false"; - // Which program the real-TUI host drives — defaults to integration. - if (opts.program) childEnv.PROGRAM = opts.program; + childEnv.PROGRAM = program; // ── Warehouse wiring: the stub MCP, the answers, the run variation ──── let stub: McpStub | null = null;