From 53491e7c984d17153e0e289f6c148d6f0b025219 Mon Sep 17 00:00:00 2001 From: Kyle Brown Date: Wed, 2 Sep 2026 22:27:43 +0000 Subject: [PATCH] refactor(examples): port custom validation to TypeScript --- .gitignore | 4 + examples/custom-validation/README.md | 19 +++-- examples/custom-validation/app.mts | 58 ++++++++++++++ examples/custom-validation/app.py | 48 ----------- examples/custom-validation/run.mjs | 25 +++++- examples/custom-validation/scan.md | 4 +- examples/custom-validation/validate.mts | 79 +++++++++++++++++++ examples/custom-validation/validate.py | 60 -------------- examples/custom-validation/validation.md | 4 +- sdk/typescript/package.json | 3 +- .../custom-validation-example.test.ts | 55 +++++++++++++ sdk/typescript/tsconfig.examples.json | 7 ++ sdk/typescript/tsconfig.json | 1 + 13 files changed, 247 insertions(+), 120 deletions(-) create mode 100644 examples/custom-validation/app.mts delete mode 100644 examples/custom-validation/app.py create mode 100644 examples/custom-validation/validate.mts delete mode 100644 examples/custom-validation/validate.py create mode 100644 sdk/typescript/tests-ts/custom-validation-example.test.ts create mode 100644 sdk/typescript/tsconfig.examples.json diff --git a/.gitignore b/.gitignore index 3c4dba3eb..8e82c49f2 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,7 @@ __pycache__/ # Emitted by sdk/typescript's build:ci script. /.github/scripts/check_plugin_source_compatibility.mjs /.github/scripts/test_check_plugin_source_compatibility.mjs + +# Emitted by sdk/typescript's build:examples script. +/examples/custom-validation/app.mjs +/examples/custom-validation/validate.mjs diff --git a/examples/custom-validation/README.md b/examples/custom-validation/README.md index 16aa0464a..571de2d87 100644 --- a/examples/custom-validation/README.md +++ b/examples/custom-validation/README.md @@ -2,8 +2,9 @@ This deliberately vulnerable invoice API contains only synthetic data. Do not deploy it. The validation script starts a real loopback HTTP server, tests -cross-account access, saves the evidence, and stops the server. It needs Python -3.10 or later and no extra packages or Docker. +cross-account access, saves the evidence, and stops the server. It uses the +SDK's supported Node.js version (including 22.13) and TypeScript compiler, with no +extra packages or Docker. From the repository root, build the CLI and run the demo: @@ -14,8 +15,16 @@ node examples/custom-validation/run.mjs ``` The runner uses your existing Codex Security sign-in or API key. It copies the -fixture to a temporary directory and prints the scan output path. Extra CLI -options can be appended, for example `--model gpt-5.6-terra --effort high`. +TypeScript fixture to a temporary directory, compiles it to JavaScript, and +prints the scan output path. Extra CLI options can be appended, for example +`--model gpt-5.6-terra --effort high`. + +To run just the HTTP proof without a scan: + +```bash +pnpm --dir sdk/typescript run build:examples +node examples/custom-validation/validate.mjs --output reports/http-proof.json +``` Look for these files in the printed scan directory: @@ -31,4 +40,4 @@ cannot complete; it does not fall back to the default validation workflow. Adapt [validation.md](validation.md) for your own setup, tests, and cleanup. For a Docker-based project, the same prompt can run your existing compose or -test script instead of `validate.py`. +test script instead of `validate.mjs`. diff --git a/examples/custom-validation/app.mts b/examples/custom-validation/app.mts new file mode 100644 index 000000000..478d252a6 --- /dev/null +++ b/examples/custom-validation/app.mts @@ -0,0 +1,58 @@ +// Deliberately vulnerable local fixture. Do not deploy this application. +import { once } from "node:events"; +import { createServer as createHttpServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { pathToFileURL } from "node:url"; + +// These identities, tokens, and records are synthetic demo data. +const tokens = new Map([ + ["demo-alice", "alice"], + ["demo-bob", "bob"], +]); +type Invoice = { id: string; owner: string; amount: number }; +const invoices = new Map([ + ["1001", { id: "1001", owner: "alice", amount: 25 }], + ["1002", { id: "1002", owner: "bob", amount: 80 }], +]); + +export async function createServer(): Promise { + const server = createHttpServer((request, response) => { + function reply(status: number, body: Invoice | { error: string }): void { + const encoded = JSON.stringify(body); + response.writeHead(status, { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(encoded), + }); + response.end(encoded); + } + + if (request.method !== "GET") { + response.writeHead(501).end(); + return; + } + const authorization = request.headers.authorization ?? ""; + const token = authorization.startsWith("Bearer ") + ? authorization.slice("Bearer ".length) + : authorization; + const user = tokens.get(token); + if (user === undefined) return reply(401, { error: "unauthorized" }); + const path = request.url ?? ""; + if (!path.startsWith("/invoices/")) + return reply(404, { error: "not found" }); + const invoice = invoices.get(path.slice("/invoices/".length)); + if (invoice === undefined) return reply(404, { error: "not found" }); + // BUG: authentication does not establish ownership of this invoice. + reply(200, invoice); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + return server; +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + const server = await createServer(); + console.log(`http://127.0.0.1:${(server.address() as AddressInfo).port}`); +} diff --git a/examples/custom-validation/app.py b/examples/custom-validation/app.py deleted file mode 100644 index b9267bdc5..000000000 --- a/examples/custom-validation/app.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Deliberately vulnerable local fixture. Do not deploy this application.""" - -import json -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - - -# These identities, tokens, and records are synthetic demo data. -TOKENS = {"demo-alice": "alice", "demo-bob": "bob"} -INVOICES = { - "1001": {"id": "1001", "owner": "alice", "amount": 25}, - "1002": {"id": "1002", "owner": "bob", "amount": 80}, -} - - -class InvoiceHandler(BaseHTTPRequestHandler): - def reply(self, status, body): - encoded = json.dumps(body).encode() - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(encoded))) - self.end_headers() - self.wfile.write(encoded) - - def do_GET(self): - token = self.headers.get("Authorization", "").removeprefix("Bearer ") - user = TOKENS.get(token) - if user is None: - return self.reply(401, {"error": "unauthorized"}) - if not self.path.startswith("/invoices/"): - return self.reply(404, {"error": "not found"}) - invoice = INVOICES.get(self.path.removeprefix("/invoices/")) - if invoice is None: - return self.reply(404, {"error": "not found"}) - # BUG: authentication does not establish ownership of this invoice. - return self.reply(200, invoice) - - def log_message(self, format, *args): - pass - - -def create_server(): - return ThreadingHTTPServer(("127.0.0.1", 0), InvoiceHandler) - - -if __name__ == "__main__": - with create_server() as server: - print(f"http://127.0.0.1:{server.server_port}", flush=True) - server.serve_forever() diff --git a/examples/custom-validation/run.mjs b/examples/custom-validation/run.mjs index fa790e04a..2eb2b5a35 100644 --- a/examples/custom-validation/run.mjs +++ b/examples/custom-validation/run.mjs @@ -8,10 +8,31 @@ const root = await mkdtemp(join(tmpdir(), "codex-security-validation-demo-")); const target = join(root, "target"); const output = join(root, "scan"); await mkdir(target); -for (const name of ["app.py", "validate.py"]) { +for (const name of ["app.mts", "validate.mts"]) { await copyFile(new URL(name, import.meta.url), join(target, name)); } +const build = spawnSync( + process.execPath, + [ + fileURLToPath( + new URL( + "../../sdk/typescript/node_modules/typescript/bin/tsc", + import.meta.url, + ), + ), + "--project", + fileURLToPath( + new URL("../../sdk/typescript/tsconfig.examples.json", import.meta.url), + ), + "--outDir", + target, + ], + { stdio: "inherit" }, +); +if (build.error) throw build.error; +if (build.status !== 0) process.exit(build.status ?? 1); + console.log(`Demo target: ${target}\nScan output: ${output}`); const child = spawnSync( process.execPath, @@ -22,7 +43,7 @@ const child = spawnSync( "scan", target, "--path", - "app.py", + "app.mts", "--scan-prompt-file", fileURLToPath(new URL("scan.md", import.meta.url)), "--validation-prompt-file", diff --git a/examples/custom-validation/scan.md b/examples/custom-validation/scan.md index 0f6e4ff6c..eb7accbf7 100644 --- a/examples/custom-validation/scan.md +++ b/examples/custom-validation/scan.md @@ -1,4 +1,4 @@ -Review invoice ownership checks in `app.py`. An authenticated account must not +Review invoice ownership checks in `app.mts`. An authenticated account must not read another account's invoice. The fixed tokens and records are synthetic test -data, not production credentials. `validate.py` is a test harness, not an +data, not production credentials. `validate.mts` is a test harness, not an application endpoint. Keep discovery source-only. diff --git a/examples/custom-validation/validate.mts b/examples/custom-validation/validate.mts new file mode 100644 index 000000000..68bfc378d --- /dev/null +++ b/examples/custom-validation/validate.mts @@ -0,0 +1,79 @@ +// Exercise the fixture over real HTTP and save the observed evidence. +import assert from "node:assert/strict"; +import { mkdir, writeFile } from "node:fs/promises"; +import type { AddressInfo } from "node:net"; +import { dirname } from "node:path"; +import { parseArgs } from "node:util"; +import { createServer } from "./app.mjs"; + +type HttpResult = { + status: number; + body: { id?: string; owner?: string; amount?: number; error?: string }; +}; + +async function main(): Promise { + let output: string; + try { + const { values } = parseArgs({ + options: { + output: { type: "string" }, + help: { type: "boolean", short: "h" }, + }, + }); + if (values.help) { + console.log("Usage: node validate.mjs --output PATH"); + return 0; + } + if (values.output === undefined) throw new Error("--output is required"); + output = values.output; + } catch (error) { + console.error((error as Error).message); + return 2; + } + + const server = await createServer(); + const baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + async function get(invoice: string, token?: string): Promise { + const response = await fetch(`${baseUrl}/invoices/${invoice}`, { + headers: token === undefined ? {} : { Authorization: `Bearer ${token}` }, + signal: AbortSignal.timeout(5_000), + }); + return { + status: response.status, + body: (await response.json()) as HttpResult["body"], + }; + } + + let evidence: { + anonymous: HttpResult; + own_invoice: HttpResult; + other_invoice: HttpResult; + cross_account_read: boolean; + }; + try { + const anonymous = await get("1002"); + const own_invoice = await get("1001", "demo-alice"); + const other_invoice = await get("1002", "demo-alice"); + assert.equal(anonymous.status, 401, "Authentication control failed"); + assert.equal(own_invoice.status, 200, "Own-account control failed"); + evidence = { + anonymous, + own_invoice, + other_invoice, + cross_account_read: + other_invoice.status === 200 && other_invoice.body.owner === "bob", + }; + } finally { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } + + const proof = { ...evidence, server_stopped: true }; + await mkdir(dirname(output), { recursive: true }); + await writeFile(output, `${JSON.stringify(proof, null, 2)}\n`, "utf8"); + console.log(JSON.stringify(proof)); + return 0; +} + +process.exitCode = await main(); diff --git a/examples/custom-validation/validate.py b/examples/custom-validation/validate.py deleted file mode 100644 index c7430a689..000000000 --- a/examples/custom-validation/validate.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Exercise the fixture over real HTTP and save the observed evidence.""" - -import argparse -import json -import runpy -from http.client import HTTPConnection -from pathlib import Path -from threading import Thread - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--output", required=True, type=Path) - args = parser.parse_args() - - app = runpy.run_path(str(Path(__file__).with_name("app.py"))) - server = app["create_server"]() - worker = Thread(target=server.serve_forever, daemon=True) - worker.start() - - def get(invoice, token=None): - connection = HTTPConnection("127.0.0.1", server.server_port, timeout=5) - try: - headers = ( - {} if token is None else {"Authorization": f"Bearer {token}"} - ) - connection.request("GET", f"/invoices/{invoice}", headers=headers) - response = connection.getresponse() - return {"status": response.status, "body": json.loads(response.read())} - finally: - connection.close() - - try: - anonymous = get("1002") - own_invoice = get("1001", "demo-alice") - other_invoice = get("1002", "demo-alice") - assert anonymous["status"] == 401, "Authentication control failed" - assert own_invoice["status"] == 200, "Own-account control failed" - evidence = { - "anonymous": anonymous, - "own_invoice": own_invoice, - "other_invoice": other_invoice, - "cross_account_read": ( - other_invoice["status"] == 200 - and other_invoice["body"].get("owner") == "bob" - ), - } - finally: - server.shutdown() - server.server_close() - worker.join() - - evidence["server_stopped"] = True - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(evidence, indent=2) + "\n", encoding="utf-8") - print(json.dumps(evidence)) - - -if __name__ == "__main__": - main() diff --git a/examples/custom-validation/validation.md b/examples/custom-validation/validation.md index 0aec3c4a4..ba2aec678 100644 --- a/examples/custom-validation/validation.md +++ b/examples/custom-validation/validation.md @@ -1,9 +1,9 @@ Validate the invoice-ownership finding against this local fixture. -1. Use the configured Python interpreter to run `validate.py` from the supplied +1. Run `node validate.mjs` from the supplied repository root. Pass `--output` with the absolute path to `artifacts/custom-validation/http-proof.json` inside this scan's directory. - Set `PYTHONDONTWRITEBYTECODE=1` so the target remains unchanged. + The runner has already compiled the TypeScript fixture to JavaScript. 2. The script starts a server on an ephemeral `127.0.0.1` port, makes three HTTP requests using synthetic identities, and shuts the server down. This local server is the only authorized test target. Do not install packages or contact diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index 90b825883..6df215058 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -50,10 +50,11 @@ "clean": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\"", "build": "node --run clean && tsc -p tsconfig.build.json && node scripts/build-dashboard.mjs", "build:ci": "tsc -p tsconfig.ci.json", + "build:examples": "tsc -p tsconfig.examples.json", "build:plugin": "node scripts/build-plugin.mjs", "check:plugin-source": "node scripts/check-plugin-source.mjs", "check:package": "node scripts/check-package.mjs", - "format": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore \"**/*.{cjs,mjs,js,ts,mts,tsx,json,md}\" \"../../.github/scripts/*.mts\"", + "format": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore \"**/*.{cjs,mjs,js,ts,mts,tsx,json,md}\" \"../../.github/scripts/*.mts\" \"../../examples/custom-validation/*.{mts,md}\"", "generate:models": "node scripts/generate-models.cjs", "generate:models:check": "node scripts/generate-models.cjs --check", "lint": "tsc --noEmit", diff --git a/sdk/typescript/tests-ts/custom-validation-example.test.ts b/sdk/typescript/tests-ts/custom-validation-example.test.ts new file mode 100644 index 000000000..d1dbfd575 --- /dev/null +++ b/sdk/typescript/tests-ts/custom-validation-example.test.ts @@ -0,0 +1,55 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, test } from "bun:test"; +import { runCommand } from "./support/shell.js"; + +test("compiled validation example saves HTTP proof and exits after server cleanup", async () => { + const source = await mkdtemp(join(tmpdir(), "custom-validation-example-")); + const output = await mkdtemp(join(tmpdir(), "custom-validation-proof-")); + const packageRoot = fileURLToPath(new URL("..", import.meta.url)); + try { + const build = await runCommand( + "node", + [ + join(packageRoot, "node_modules", "typescript", "bin", "tsc"), + "--project", + join(packageRoot, "tsconfig.examples.json"), + "--outDir", + source, + ], + { timeout: 30_000 }, + ); + expect(build.status, build.stdout + build.stderr).toBe(0); + const proofPath = join(output, "artifacts", "http-proof.json"); + const result = await runCommand( + "node", + [join(source, "validate.mjs"), "--output", proofPath], + { timeout: 30_000 }, + ); + // A server left listening would prevent this process from exiting normally. + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toBe(""); + const proof = { + anonymous: { status: 401, body: { error: "unauthorized" } }, + own_invoice: { + status: 200, + body: { id: "1001", owner: "alice", amount: 25 }, + }, + other_invoice: { + status: 200, + body: { id: "1002", owner: "bob", amount: 80 }, + }, + cross_account_read: true, + server_stopped: true, + }; + expect(JSON.parse(result.stdout)).toEqual(proof); + expect(JSON.parse(await readFile(proofPath, "utf8"))).toEqual(proof); + } finally { + await Promise.all([ + rm(source, { recursive: true, force: true }), + rm(output, { recursive: true, force: true }), + ]); + } +}); diff --git a/sdk/typescript/tsconfig.examples.json b/sdk/typescript/tsconfig.examples.json new file mode 100644 index 000000000..eabb66409 --- /dev/null +++ b/sdk/typescript/tsconfig.examples.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.ci.json", + "include": ["../../examples/custom-validation/*.mts"], + "compilerOptions": { + "rootDir": "../../examples/custom-validation" + } +} diff --git a/sdk/typescript/tsconfig.json b/sdk/typescript/tsconfig.json index 6738b4dcf..1b08f5158 100644 --- a/sdk/typescript/tsconfig.json +++ b/sdk/typescript/tsconfig.json @@ -6,6 +6,7 @@ "dashboard/**/*.tsx", "tests-ts/**/*.ts", "../../.github/scripts/*.mts", + "../../examples/custom-validation/*.mts", "scripts/compare-test-reports.mts", "scripts/smoke-findings-service.ts", "scripts/fixtures/findings-service-sqlite.ts",