diff --git a/.gitattributes b/.gitattributes index a411474ea..e9a5ecc5b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,6 +3,7 @@ *.json text eol=lf *.md text eol=lf *.mjs text eol=lf +*.mts text eol=lf *.py text eol=lf *.sh text eol=lf *.ts text eol=lf diff --git a/.github/scripts/check_plugin_source_compatibility.mts b/.github/scripts/check_plugin_source_compatibility.mts new file mode 100644 index 000000000..5fcd6b53f --- /dev/null +++ b/.github/scripts/check_plugin_source_compatibility.mts @@ -0,0 +1,165 @@ +#!/usr/bin/env node +// Check that tracked plugin source stays portable across repository imports. + +import { spawnSync } from "node:child_process"; +import { lstatSync, readFileSync } from "node:fs"; +import { basename, extname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; + +const MAX_SOURCE_FILE_BYTES = 150_000; +const MAX_DEPENDENCY_LOCK_BYTES = 2_000_000; +const DEPENDENCY_LOCK_NAMES = new Set([ + "Cargo.lock", + "package-lock.json", + "pnpm-lock.yaml", + "requirements.txt", + "uv.lock", + "yarn.lock", +]); +const LIST_ITEM = /^\s*(?:[-*+]|\d+[.)])\s+/u; +const HTML_BLOCK = /^\s*<\/?[A-Za-z][^>]*>\s*$/u; +const NATURAL_LINE_ENDINGS = new Set("\\.?!:;。!?:;)]}'\"`>"); +const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + +function trackedFiles(pluginRoot: string): string[] { + const result = spawnSync( + "git", + ["-C", pluginRoot, "ls-files", "-z", "--", "."], + { + maxBuffer: Infinity, + }, + ); + if (result.error) throw result.error; + if (result.status !== 0) { + throw new Error(result.stderr.toString().trim() || "git ls-files failed"); + } + return utf8.decode(result.stdout).split("\0").filter(Boolean); +} + +function isMarkdownStructure(line: string): boolean { + const stripped = line.trim(); + return ( + !stripped || + ["#", ">", "|", "\n::directive\n
\n code\n\tmore code\n", + "A list\n- first\n1. second\n2) third\n", + "Complete.\nQuestion?\nBang!\nColon:\nSemicolon;\n。\n!\n?\n:\n;\nParen)\nBracket]\nBrace}\nQuote'\nDouble\"\nCode`\nAngle>\nBackslash\\\nBreak \nhttps://example.invalid/url\nlast\n", + ].map((content, index) => [`example-${index}.md`, content]), + ), + ); + const result = runChecker("--plugin-root", root); + assert.equal(result.status, 0, result.stderr); +}); + +test("rejects hard wraps after closed front matter and fences, including CRLF", () => { + const root = fixture({ + "README.MD": + "---\r\ntitle: Example\r\n---\r\n```\r\ncode\r\n```\r\nThis continues\r\non another line.\r\n", + }); + const result = runChecker("--plugin-root", root); + assert.equal(result.status, 1); + assert.match(result.stderr, /README\.MD:7: prose is hard-wrapped/u); +}); + +test("rejects dependency lock files above two megabytes", () => { + const root = fixture({ "pnpm-lock.yaml": "x".repeat(2_000_001) }); + const result = runChecker("--plugin-root", root); + assert.equal(result.status, 1); + assert.equal( + result.stderr, + "pnpm-lock.yaml: file is 2000001 bytes; maximum is 2000000 bytes\n", + ); +}); + +test( + "does not follow tracked symlinks outside the plugin", + { skip: process.platform === "win32" }, + () => { + const outside = fixture({ + "outside.md": "This outside prose continues\nonto another source line.\n", + }); + const root = fixture(); + symlinkSync(join(outside, "outside.md"), join(root, "linked.md")); + git(root, "add", "--", "linked.md"); + const result = runChecker("--plugin-root", root); + assert.equal(result.status, 0, result.stderr); + }, +); + +test("reports Git, missing tracked files, and invalid UTF-8 as check failures", () => { + const root = fixture({ "README.md": Buffer.from([0xff]) }); + for (const action of [ + () => {}, + () => unlinkSync(join(root, "README.md")), + () => rmSync(join(root, ".git"), { recursive: true }), + ]) { + action(); + const result = runChecker("--plugin-root", root); + assert.equal(result.status, 2); + assert.match(result.stderr, /^source compatibility check failed:/u); + } +}); + +test("help describes the source contract and invalid arguments fail", () => { + const result = runChecker("--help"); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /tracked plugin source/u); + assert.equal(runChecker("--plugin-root").status, 2); + assert.equal(runChecker("--unknown").status, 2); +}); diff --git a/.github/scripts/test_check_plugin_source_compatibility.py b/.github/scripts/test_check_plugin_source_compatibility.py deleted file mode 100644 index f49a80420..000000000 --- a/.github/scripts/test_check_plugin_source_compatibility.py +++ /dev/null @@ -1,159 +0,0 @@ -from __future__ import annotations - -import os -import subprocess -import sys -from pathlib import Path - -import pytest - -CHECKER = Path(__file__).with_name("check_plugin_source_compatibility.py") - - -def initialize_repository(root: Path) -> None: - subprocess.run(["git", "init", "--quiet", str(root)], check=True) - - -def track(root: Path, *paths: str) -> None: - subprocess.run(["git", "-C", str(root), "add", "--", *paths], check=True) - - -def run_checker(root: Path, *args: str) -> subprocess.CompletedProcess[str]: - return subprocess.run( - [sys.executable, str(CHECKER), "--plugin-root", str(root), *args], - check=False, - capture_output=True, - text=True, - ) - - -def test_reports_tracked_source_violations_in_stable_order(tmp_path: Path) -> None: - (tmp_path / "notes.md").write_text( - "This prose continues in the middle of a sentence\nonto another source line.\n", - encoding="utf-8", - ) - (tmp_path / "oversized.py").write_bytes(b"x" * 150_001) - initialize_repository(tmp_path) - track(tmp_path, "oversized.py", "notes.md") - - result = run_checker(tmp_path) - - assert result.returncode == 1 - assert result.stdout == "" - assert result.stderr.splitlines() == [ - "notes.md:1: prose is hard-wrapped mid-sentence; use a natural Markdown line", - "oversized.py: file is 150001 bytes; maximum is 150000 bytes", - ] - - -def test_accepts_valid_source_and_ignores_untracked_files(tmp_path: Path) -> None: - (tmp_path / "README.md").write_text("A complete sentence.\n", encoding="utf-8") - (tmp_path / "package-lock.json").write_bytes(b"x" * 150_001) - (tmp_path / "untracked.md").write_text( - "This untracked prose continues\nonto another source line.\n", - encoding="utf-8", - ) - initialize_repository(tmp_path) - track(tmp_path, "README.md", "package-lock.json") - - result = run_checker(tmp_path) - - assert result.returncode == 0, result.stderr - assert result.stdout == "Plugin source compatibility checks passed.\n" - assert result.stderr == "" - - -def test_python_checkout_preserves_source_size_with_autocrlf(tmp_path: Path) -> None: - attributes = CHECKER.parents[2] / ".gitattributes" - (tmp_path / ".gitattributes").write_bytes(attributes.read_bytes()) - source = tmp_path / "module.py" - content = b"pass\n" * 30_000 - source.write_bytes(content) - initialize_repository(tmp_path) - subprocess.run( - ["git", "-C", str(tmp_path), "config", "--local", "core.autocrlf", "true"], - check=True, - ) - track(tmp_path, ".gitattributes", "module.py") - source.unlink() - subprocess.run( - ["git", "-C", str(tmp_path), "checkout-index", "--", "module.py"], - check=True, - ) - - result = run_checker(tmp_path) - - assert result.returncode == 0, result.stderr - assert source.read_bytes() == content - - -def test_accepts_prose_after_an_opening_thematic_break(tmp_path: Path) -> None: - (tmp_path / "README.md").write_text( - """--- -This prose continues -onto another source line. -""", - encoding="utf-8", - ) - initialize_repository(tmp_path) - track(tmp_path, "README.md") - - result = run_checker(tmp_path) - - assert result.returncode == 0, result.stderr - - -@pytest.mark.parametrize( - "content", - [ - "First clause,\ncontinues here.\n", - "First clause\n**continues** here.\n", - ], -) -def test_accepts_wraps_adjacent_to_inline_markup(tmp_path: Path, content: str) -> None: - (tmp_path / "README.md").write_text(content, encoding="utf-8") - initialize_repository(tmp_path) - track(tmp_path, "README.md") - - result = run_checker(tmp_path) - - assert result.returncode == 0, result.stderr - - -def test_rejects_dependency_lock_files_above_two_megabytes(tmp_path: Path) -> None: - (tmp_path / "pnpm-lock.yaml").write_bytes(b"x" * 2_000_001) - initialize_repository(tmp_path) - track(tmp_path, "pnpm-lock.yaml") - - result = run_checker(tmp_path) - - assert result.returncode == 1 - assert result.stderr == ("pnpm-lock.yaml: file is 2000001 bytes; maximum is 2000000 bytes\n") - - -@pytest.mark.skipif(os.name == "nt", reason="creating symlinks requires elevated Windows access") -def test_does_not_follow_tracked_symlinks_outside_the_plugin(tmp_path: Path) -> None: - outside = tmp_path.parent / f"{tmp_path.name}-outside.md" - outside.write_text( - "This outside prose continues\nonto another source line.\n", - encoding="utf-8", - ) - (tmp_path / "linked.md").symlink_to(outside) - initialize_repository(tmp_path) - track(tmp_path, "linked.md") - - result = run_checker(tmp_path) - - assert result.returncode == 0, result.stderr - - -def test_help_describes_the_source_contract() -> None: - result = subprocess.run( - [sys.executable, str(CHECKER), "--help"], - check=False, - capture_output=True, - text=True, - ) - - assert result.returncode == 0, result.stderr - assert "tracked plugin source" in result.stdout diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml index 07108c904..8ae403fb4 100644 --- a/.github/workflows/node-ci.yml +++ b/.github/workflows/node-ci.yml @@ -82,11 +82,6 @@ jobs: exit 1 fi - - name: Check plugin source compatibility - if: steps.scope.outputs.ci-mode == 'markdown' - run: | - python .github/scripts/check_plugin_source_compatibility.py - - name: Set up pnpm if: steps.scope.outputs.check-markdown == 'true' uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 @@ -105,6 +100,15 @@ jobs: if: steps.scope.outputs.check-markdown == 'true' run: pnpm --dir sdk/typescript install --frozen-lockfile + - name: Compile CI scripts + if: steps.scope.outputs.ci-mode == 'markdown' + run: pnpm --dir sdk/typescript run build:ci + + - name: Check plugin source compatibility + if: steps.scope.outputs.ci-mode == 'markdown' + run: | + node .github/scripts/check_plugin_source_compatibility.mjs + - name: Check Markdown formatting if: steps.scope.outputs.check-markdown == 'true' shell: bash @@ -362,6 +366,20 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - name: Set up Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: "22.13.0" + - name: Set up pnpm + uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + with: + package_json_file: package.json + cache: true + cache_dependency_path: sdk/typescript/pnpm-lock.yaml + - name: Install dependencies + run: pnpm --dir sdk/typescript install --frozen-lockfile + - name: Compile CI scripts + run: pnpm --dir sdk/typescript run build:ci - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: @@ -376,11 +394,13 @@ jobs: sudo apt-get install --yes ripgrep - name: Check plugin source compatibility run: | - python -m ruff check --config plugins/codex-security/pyproject.toml .github/scripts/check_plugin_source_compatibility.py .github/scripts/test_check_plugin_source_compatibility.py plugins/codex-security - python -m ruff format --check --config plugins/codex-security/pyproject.toml .github/scripts/check_plugin_source_compatibility.py .github/scripts/test_check_plugin_source_compatibility.py plugins/codex-security - python .github/scripts/check_plugin_source_compatibility.py + python -m ruff check --config plugins/codex-security/pyproject.toml plugins/codex-security + python -m ruff format --check --config plugins/codex-security/pyproject.toml plugins/codex-security + node .github/scripts/check_plugin_source_compatibility.mjs + - name: Test source compatibility checker + run: node --test .github/scripts/test_check_plugin_source_compatibility.mjs - name: Test Python source contracts - run: python -m pytest .github/scripts/test_check_plugin_source_compatibility.py plugins/codex-security/tests -q -n 4 --dist worksteal --max-worker-restart 0 --durations=30 --junitxml=reports/python.xml + run: python -m pytest plugins/codex-security/tests -q -n 4 --dist worksteal --max-worker-restart 0 --durations=30 --junitxml=reports/python.xml - name: Upload Python test reports if: always() continue-on-error: true @@ -391,10 +411,6 @@ jobs: path: reports/python.xml if-no-files-found: warn retention-days: 14 - - name: Set up Node.js - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 - with: - node-version: "22.13.0" - name: Test deterministic triage eval contracts working-directory: plugins/codex-security/skills/triage-finding/evals run: node --run test:deterministic diff --git a/.github/workflows/test-quality.yml b/.github/workflows/test-quality.yml index 1400b02bc..859a25c20 100644 --- a/.github/workflows/test-quality.yml +++ b/.github/workflows/test-quality.yml @@ -113,6 +113,18 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: "22.13.0" + - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + with: + package_json_file: package.json + cache: true + cache_dependency_path: sdk/typescript/pnpm-lock.yaml + - name: Install comparison dependencies + run: pnpm --dir sdk/typescript install --frozen-lockfile + - name: Compile CI scripts + run: pnpm --dir sdk/typescript run build:ci - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: runner-* @@ -124,10 +136,10 @@ jobs: comparison_status=0 for os in ubuntu-latest windows-latest; do for mode in isolated parallel; do - python3 sdk/typescript/scripts/compare-test-reports.py "reports/runner-$os-baseline.xml" "reports/runner-$os-$mode.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1 + node sdk/typescript/scripts/compare-test-reports.mjs "reports/runner-$os-baseline.xml" "reports/runner-$os-$mode.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1 done done - python3 sdk/typescript/scripts/compare-test-reports.py reports/runner-windows-latest-baseline.xml 'reports/runner-windows-latest-shard-*.xml' >> "$GITHUB_STEP_SUMMARY" || comparison_status=1 + node sdk/typescript/scripts/compare-test-reports.mjs reports/runner-windows-latest-baseline.xml 'reports/runner-windows-latest-shard-*.xml' >> "$GITHUB_STEP_SUMMARY" || comparison_status=1 exit "$comparison_status" mutation: diff --git a/.gitignore b/.gitignore index d6b8b94b5..3c4dba3eb 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,7 @@ __pycache__/ # Never publish internal-only plugin data. .internal/ + +# Emitted by sdk/typescript's build:ci script. +/.github/scripts/check_plugin_source_compatibility.mjs +/.github/scripts/test_check_plugin_source_compatibility.mjs diff --git a/AGENTS.md b/AGENTS.md index a395c4c4a..a8a5b1fe6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,9 +18,11 @@ When changing `plugins/codex-security`, run its portable source checks before submitting the change: ```bash -python -m ruff check --config plugins/codex-security/pyproject.toml .github/scripts/check_plugin_source_compatibility.py .github/scripts/test_check_plugin_source_compatibility.py plugins/codex-security -python -m ruff format --check --config plugins/codex-security/pyproject.toml .github/scripts/check_plugin_source_compatibility.py .github/scripts/test_check_plugin_source_compatibility.py plugins/codex-security -python .github/scripts/check_plugin_source_compatibility.py +python -m ruff check --config plugins/codex-security/pyproject.toml plugins/codex-security +python -m ruff format --check --config plugins/codex-security/pyproject.toml plugins/codex-security +pnpm --dir sdk/typescript run build:ci +node .github/scripts/check_plugin_source_compatibility.mjs +node --test .github/scripts/test_check_plugin_source_compatibility.mjs ``` ## Avoid speculative defenses diff --git a/sdk/typescript/.gitignore b/sdk/typescript/.gitignore index 973a09c34..3dd160171 100644 --- a/sdk/typescript/.gitignore +++ b/sdk/typescript/.gitignore @@ -6,3 +6,4 @@ /.stryker-tmp/ /private_release/dist/ /*.tsbuildinfo +/scripts/compare-test-reports.mjs diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index 0b0889e7e..733f7b733 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -44,10 +44,11 @@ "audit:prod": "pnpm audit --prod --audit-level high", "clean": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\"", "build": "node --run clean && tsc -p tsconfig.build.json && node scripts/build-dashboard.mjs", + "build:ci": "tsc -p tsconfig.ci.json", "build:plugin": "node scripts/build-plugin.mjs", "check:plugin-source": "node scripts/check-plugin-source.mjs", "check:package": "node scripts/check-package.mjs", - "format": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore \"**/*.{cjs,mjs,js,ts,tsx,json,md}\"", + "format": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore \"**/*.{cjs,mjs,js,ts,mts,tsx,json,md}\" \"../../.github/scripts/*.mts\"", "generate:models": "node scripts/generate-models.cjs", "generate:models:check": "node scripts/generate-models.cjs --check", "lint": "tsc --noEmit", @@ -93,9 +94,11 @@ "fast-check": "4.9.0", "ink-testing-library": "4.0.0", "json-schema-to-typescript": "15.0.4", + "minimatch": "10.2.6", "postcss": "8.5.23", "prettier": "3.2.5", "react-dom": "19.2.4", + "saxes": "6.0.0", "tailwindcss": "4.3.3", "typescript": "5.7.3" } diff --git a/sdk/typescript/pnpm-lock.yaml b/sdk/typescript/pnpm-lock.yaml index eed94b772..b38d32fc6 100644 --- a/sdk/typescript/pnpm-lock.yaml +++ b/sdk/typescript/pnpm-lock.yaml @@ -99,6 +99,9 @@ importers: json-schema-to-typescript: specifier: 15.0.4 version: 15.0.4 + minimatch: + specifier: 10.2.6 + version: 10.2.6 postcss: specifier: 8.5.23 version: 8.5.23 @@ -108,6 +111,9 @@ importers: react-dom: specifier: 19.2.4 version: 19.2.4(react@19.2.4) + saxes: + specifier: 6.0.0 + version: 6.0.0 tailwindcss: specifier: 4.3.3 version: 4.3.3 @@ -2682,6 +2688,10 @@ packages: safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + saxes@6.0.0: + resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==} + engines: {node: '>=v12.22.7'} + scheduler@0.27.0: resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==} @@ -2946,6 +2956,9 @@ packages: utf-8-validate: optional: true + xmlchars@2.2.0: + resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} + yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} @@ -5897,6 +5910,10 @@ snapshots: safer-buffer@2.1.2: {} + saxes@6.0.0: + dependencies: + xmlchars: 2.2.0 + scheduler@0.27.0: {} semver@6.3.1: {} @@ -6145,6 +6162,8 @@ snapshots: ws@8.21.3: {} + xmlchars@2.2.0: {} + yallist@3.1.1: {} yaml@2.9.0: {} diff --git a/sdk/typescript/scripts/compare-test-reports.mts b/sdk/typescript/scripts/compare-test-reports.mts new file mode 100644 index 000000000..e78b6103c --- /dev/null +++ b/sdk/typescript/scripts/compare-test-reports.mts @@ -0,0 +1,252 @@ +// Compare Bun JUnit inventories before changing the required CI runner. +import { lstatSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { basename, dirname, join, sep } from "node:path"; +import { parseArgs } from "node:util"; +import { minimatch } from "minimatch"; +import { SaxesParser, type SaxesTagNS } from "saxes"; + +type TestStatus = "passed" | "skipped" | "failed"; +type TestIdentity = [file: string, classname: string, name: string]; +type TestOutcome = [...TestIdentity, status: TestStatus]; +type TestRecord = { + attributes: SaxesTagNS["attributes"]; + status: TestStatus; +}; +type TestReport = { + cases: Map; + duration: number; + failed: boolean; +}; + +function matchingReports(pattern: string): string[] { + // Python glob treats ** as one component and does not expand braces/extglobs. + const directoriesOnly = + pattern.endsWith(sep) || + (process.platform === "win32" && pattern.endsWith("/")); + if (!/[*?[]/u.test(pattern)) { + try { + const stat = directoriesOnly ? statSync(pattern) : lstatSync(pattern); + return !directoriesOnly || stat.isDirectory() ? [pattern] : []; + } catch { + return []; + } + } + const parent = dirname(pattern); + const namePattern = basename(pattern); + const directories = /[*?[]/u.test(parent) + ? matchingReports(parent) + : [parent]; + return directories.flatMap((directory) => { + let names: string[]; + try { + names = readdirSync(directory); + } catch { + return []; + } + return names + .filter( + (name) => + (!name.startsWith(".") || namePattern.startsWith(".")) && + minimatch( + name, + namePattern.replaceAll("\\", "\\\\").replaceAll("[^", "[\\^"), + { + dot: true, + nobrace: true, + noext: true, + noglobstar: true, + nonegate: true, + nocomment: true, + nocase: process.platform === "win32", + }, + ), + ) + .map((name) => join(directory, name)) + .filter((path) => { + if (!directoriesOnly) return true; + try { + return statSync(path).isDirectory(); + } catch { + return false; + } + }); + }); +} + +function integer(value: string): bigint { + if (!/^[+-]?\d(?:_?\d)*$/u.test(value.trim())) { + throw new Error(`invalid integer: ${value}`); + } + return BigInt(value.replaceAll("_", "").trim()); +} + +function seconds(value: string): number { + const number = value.trim().replaceAll(/(?<=\d)_(?=\d)/gu, ""); + if ( + !/^[+-]?(?:(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?|inf(?:inity)?|nan)$/iu.test( + number, + ) + ) { + throw new Error(`invalid duration: ${value}`); + } + return Number(number.replace(/inf(?:inity)?/iu, "Infinity")); +} + +function readReport(path: string): TestReport { + const parser = new SaxesParser({ xmlns: true, fileName: path }); + let root: SaxesTagNS | undefined; + const stack: Array<{ record: TestRecord | undefined }> = []; + const records: TestRecord[] = []; + const suites: SaxesTagNS[] = []; + parser.on("opentag", (node) => { + root ??= node; + const name = node.uri ? `{${node.uri}}${node.local}` : node.local; + const parent = stack.at(-1); + if (parent?.record) { + if (name === "failure" || name === "error") + parent.record.status = "failed"; + else if (name === "skipped" && parent.record.status === "passed") + parent.record.status = "skipped"; + } + let record: TestRecord | undefined; + if (name === "testcase") { + record = { attributes: node.attributes, status: "passed" }; + records.push(record); + } + if (name === "testsuite" || name === "testsuites") suites.push(node); + stack.push({ record }); + }); + parser.on("closetag", () => stack.pop()); + const content = new TextDecoder("utf-8", { fatal: true }).decode( + readFileSync(path), + ); + parser.write(content).close(); + + const cases = new Map(); + for (const { attributes, status } of records) { + const identity: TestIdentity = [ + (attributes["file"]?.value ?? "") + .replaceAll("\\", "/") + .replace(/^\.\//u, ""), + attributes["classname"]?.value ?? "", + attributes["name"]?.value ?? "", + ]; + const key = JSON.stringify(identity); + if (cases.has(key)) + throw new Error( + `${path}: duplicate test identity: ${identity.join(" > ")}`, + ); + cases.set(key, status); + } + if (!cases.size) throw new Error(`${path}: no test cases`); + if ( + integer(root!.attributes["tests"]?.value ?? String(cases.size)) !== + BigInt(cases.size) + ) { + throw new Error(`${path}: reported test count does not match test cases`); + } + const failed = + [...cases.values()].includes("failed") || + suites.some((node) => + ["failures", "errors"].some( + (field) => integer(node.attributes[field]?.value ?? "0") !== 0n, + ), + ); + if (failed) console.error(`${path}: test run failed`); + const duration = seconds(root!.attributes["time"]?.value ?? "0"); + const skipped = [...cases.values()].filter( + (status) => status === "skipped", + ).length; + console.log( + `| ${basename(path)} | ${cases.size} | ${skipped} | ${duration.toFixed(2)} |`, + ); + return { + cases: new Map( + [...cases].map(([identity, status]) => [ + JSON.stringify([...(JSON.parse(identity) as TestIdentity), status]), + 1, + ]), + ), + duration, + failed, + }; +} + +function main(): number { + let args: { values: { help?: boolean }; positionals: string[] }; + try { + args = parseArgs({ + options: { help: { type: "boolean", short: "h" } }, + allowPositionals: true, + }); + if (!args.values.help && args.positionals.length < 2) + throw new Error( + "a baseline and at least one candidate report are required", + ); + } catch (error) { + console.error((error as Error).message); + return 2; + } + if (args.values.help) { + console.log( + "Compare Bun JUnit inventories before changing the required CI runner.\n\nUsage: node compare-test-reports.mjs baseline candidates [candidates ...]\nCandidates are JUnit files or glob patterns.", + ); + return 0; + } + console.log("| Report | Cases | Skipped | Seconds |"); + console.log("| --- | ---: | ---: | ---: |"); + const baseline = readReport(args.positionals[0]!); + let failed = baseline.failed; + const candidates = new Map(); + const durations: number[] = []; + for (const pattern of args.positionals.slice(1)) { + const paths = matchingReports(pattern).sort((a, b) => + Buffer.compare(Buffer.from(a), Buffer.from(b)), + ); + if (!paths.length) throw new Error(`No reports match ${pattern}`); + for (const path of paths) { + const report = readReport(path); + failed ||= report.failed; + for (const [identity, count] of report.cases) + candidates.set(identity, (candidates.get(identity) ?? 0) + count); + durations.push(report.duration); + } + } + for (const [label, left, right] of [ + ["Missing", baseline.cases, candidates], + ["Extra", candidates, baseline.cases], + ] as const) { + const differences = [...left] + .map(([identity, count]): [TestOutcome, number] => [ + JSON.parse(identity) as TestOutcome, + count - (right.get(identity) ?? 0), + ]) + .filter(([, count]) => count > 0) + .sort(([a], [b]) => { + for (let index = 0; index < a.length; index++) { + const order = Buffer.compare( + Buffer.from(a[index]!), + Buffer.from(b[index]!), + ); + if (order) return order; + } + return 0; + }); + for (const [identity, count] of differences) { + failed = true; + console.error(`${label} (${count}): ${identity.join(" > ")}`); + } + } + if (failed) return 1; + console.log( + `\nIdentical test inventory and outcomes. Slowest candidate: ${Math.max(...durations).toFixed(2)}s; combined test time: ${durations.reduce((sum, duration) => sum + duration, 0).toFixed(2)}s.\n`, + ); + return 0; +} + +try { + process.exitCode = main(); +} catch (error) { + console.error((error as Error).message); + process.exitCode = 1; +} diff --git a/sdk/typescript/scripts/compare-test-reports.py b/sdk/typescript/scripts/compare-test-reports.py deleted file mode 100644 index 6f735eed4..000000000 --- a/sdk/typescript/scripts/compare-test-reports.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Compare Bun JUnit inventories before changing the required CI runner.""" - -import argparse -from collections import Counter -from glob import glob -from pathlib import Path -import sys -import xml.etree.ElementTree as ET - - -def read_report(path: Path) -> tuple[Counter, float, bool]: - root = ET.parse(path).getroot() - cases = {} - for case in root.iter("testcase"): - status = "passed" - if case.find("skipped") is not None: - status = "skipped" - if case.find("failure") is not None or case.find("error") is not None: - status = "failed" - identity = ( - case.get("file", "").replace("\\", "/").removeprefix("./"), - case.get("classname", ""), - case.get("name", ""), - ) - if identity in cases: - raise ValueError(f"{path}: duplicate test identity: {' > '.join(identity)}") - cases[identity] = status - if not cases: - raise ValueError(f"{path}: no test cases") - if int(root.get("tests", str(len(cases)))) != len(cases): - raise ValueError(f"{path}: reported test count does not match test cases") - failed = "failed" in cases.values() or any( - int(node.get(field, "0")) - for node in root.iter() - if node.tag in ("testsuite", "testsuites") - for field in ("failures", "errors") - ) - if failed: - print(f"{path}: test run failed", file=sys.stderr) - seconds = float(root.get("time", "0")) - skipped = sum(status == "skipped" for status in cases.values()) - print(f"| {path.name} | {len(cases)} | {skipped} | {seconds:.2f} |") - return ( - Counter((*identity, status) for identity, status in cases.items()), - seconds, - failed, - ) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("baseline", type=Path) - parser.add_argument("candidates", nargs="+", help="JUnit files or glob patterns") - args = parser.parse_args() - print("| Report | Cases | Skipped | Seconds |") - print("| --- | ---: | ---: | ---: |") - baseline, _, failed = read_report(args.baseline) - candidates = Counter() - durations = [] - for pattern in args.candidates: - paths = sorted(glob(pattern)) - if not paths: - raise ValueError(f"No reports match {pattern}") - for path in paths: - cases, seconds, report_failed = read_report(Path(path)) - failed = failed or report_failed - candidates.update(cases) - durations.append(seconds) - missing, extra = baseline - candidates, candidates - baseline - if failed or missing or extra: - for label, difference in (("Missing", missing), ("Extra", extra)): - for identity, count in sorted(difference.items()): - print(f"{label} ({count}): {' > '.join(identity)}", file=sys.stderr) - return 1 - print(f"\nIdentical test inventory and outcomes. Slowest candidate: {max(durations):.2f}s; combined test time: {sum(durations):.2f}s.\n") - return 0 - - -if __name__ == "__main__": - try: - sys.exit(main()) - except (OSError, ValueError, ET.ParseError) as error: - print(error, file=sys.stderr) - sys.exit(1) diff --git a/sdk/typescript/tests-ts/release-automation.test.ts b/sdk/typescript/tests-ts/release-automation.test.ts index 2b6dde7ab..efa925bbb 100644 --- a/sdk/typescript/tests-ts/release-automation.test.ts +++ b/sdk/typescript/tests-ts/release-automation.test.ts @@ -4164,23 +4164,12 @@ describe("GitHub release workflow safeguards", () => { test.skipIf(process.platform === "win32")( "rejects oversized plugin Markdown in reduced CI", - () => { + async () => { const workspace = mkdtempSync( join(tmpdir(), "release-ci-plugin-source-"), ); const pluginRoot = join(workspace, "plugins", "codex-security"); - const scripts = join(workspace, ".github", "scripts"); - mkdirSync(scripts, { recursive: true }); mkdirSync(pluginRoot, { recursive: true }); - writeFileSync( - join(scripts, "check_plugin_source_compatibility.py"), - readFileSync( - new URL( - "../../../.github/scripts/check_plugin_source_compatibility.py", - import.meta.url, - ), - ), - ); writeFileSync(join(pluginRoot, "README.md"), "x".repeat(150_001)); spawnSync("git", ["init", "--quiet", workspace]); spawnSync("git", [ @@ -4188,10 +4177,22 @@ describe("GitHub release workflow safeguards", () => { workspace, "add", "--", - ".github/scripts/check_plugin_source_compatibility.py", "plugins/codex-security/README.md", ]); try { + const packageRoot = fileURLToPath(new URL("..", import.meta.url)); + const build = await runCommand( + "node", + [ + join(packageRoot, "node_modules", "typescript", "bin", "tsc"), + "--project", + join(packageRoot, "tsconfig.ci.json"), + "--outDir", + workspace, + ], + { timeout: 30_000 }, + ); + expect(build.status, build.stdout + build.stderr).toBe(0); const result = spawnSync( bash, [ diff --git a/sdk/typescript/tests-ts/test-reports.test.ts b/sdk/typescript/tests-ts/test-reports.test.ts index d85fba7bf..8fb83daee 100644 --- a/sdk/typescript/tests-ts/test-reports.test.ts +++ b/sdk/typescript/tests-ts/test-reports.test.ts @@ -1,11 +1,50 @@ -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { + mkdir, + mkdtemp, + readFile, + rm, + symlink, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; -import { afterEach, describe, expect, test } from "bun:test"; -import { bashCommand } from "./support/shell.js"; +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + test, +} from "bun:test"; +import { bashCommand, runCommand } from "./support/shell.js"; const bash = bashCommand(); +const packageRoot = fileURLToPath(new URL("..", import.meta.url)); +let buildRoot: string; +beforeAll(async () => { + buildRoot = await mkdtemp(join(tmpdir(), "codex-security-ci-build-")); + const result = await runCommand( + "node", + [ + join(packageRoot, "node_modules", "typescript", "bin", "tsc"), + "--project", + join(packageRoot, "tsconfig.ci.json"), + "--outDir", + buildRoot, + ], + { timeout: 30_000 }, + ); + expect(result.status, result.stdout + result.stderr).toBe(0); + await symlink( + join(packageRoot, "node_modules"), + join(buildRoot, "sdk", "typescript", "node_modules"), + process.platform === "win32" ? "junction" : "dir", + ); +}, 30_000); +afterAll(async () => { + if (buildRoot) await rm(buildRoot, { recursive: true, force: true }); +}); const directories: string[] = []; afterEach(async () => { await Promise.all( @@ -41,15 +80,15 @@ function testcase(name: string, status = "") { } async function compare(baseline: string, ...candidates: string[]) { - const python = Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py"); - if (python === null) throw new Error("A Python interpreter is required."); const child = Bun.spawn({ cmd: [ - python, - "-I", - "-B", - fileURLToPath( - new URL("../scripts/compare-test-reports.py", import.meta.url), + "node", + join( + buildRoot, + "sdk", + "typescript", + "scripts", + "compare-test-reports.mjs", ), baseline, ...candidates, @@ -88,7 +127,7 @@ describe("JUnit inventory comparison", () => { ), "reports/runner-windows-latest-shard-*.xml", ]; - const mock = `python3() { + const mock = `node() { printf '%s\\n' "$3" [[ "$3" != "$CODEX_SECURITY_TEST_FAIL_REPORT" ]] }`; @@ -128,7 +167,150 @@ describe("JUnit inventory comparison", () => { await fixture.report("shard-2.xml", [passed]); const result = await compare(baseline, join(fixture.root, "shard-*.xml")); expect(result.status, result.stderr).toBe(0); - expect(result.stdout).toContain("combined test time: 2.50s"); + expect(result.stderr).toBe(""); + expect(result.stdout).toBe( + "| Report | Cases | Skipped | Seconds |\n" + + "| --- | ---: | ---: | ---: |\n" + + "| baseline.xml | 2 | 1 | 1.25 |\n" + + "| shard-1.xml | 1 | 1 | 1.25 |\n" + + "| shard-2.xml | 1 | 0 | 1.25 |\n" + + "\nIdentical test inventory and outcomes. Slowest candidate: 1.25s; combined test time: 2.50s.\n\n", + ); + }); + + test("parses XML entities while ignoring comments and CDATA markup", async () => { + const fixture = await fixtures(); + const baseline = await fixture.report("baseline.xml", [ + testcase("a & " ' < >"), + ]); + const candidate = join(fixture.root, "candidate.xml"); + await writeFile( + candidate, + ` + + +]]> +${testcase("a & " ' < >")} +`, + ); + const result = await compare(baseline, candidate); + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toBe(""); + }); + + test("normalizes report paths and rejects duplicates across candidate files", async () => { + const fixture = await fixtures(); + const original = testcase("portable"); + const baseline = await fixture.report("baseline.xml", [original]); + const candidate = await fixture.report("candidate.xml", [ + original.replace( + "tests-ts/example.test.ts", + "./tests-ts\\example.test.ts", + ), + ]); + expect((await compare(baseline, candidate)).status).toBe(0); + const result = await compare(baseline, candidate, candidate); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "Extra (1): tests-ts/example.test.ts > example > portable > passed", + ); + }); + + test("preserves file, character-class, hidden-file, and nonrecursive glob matching", async () => { + const fixture = await fixtures(); + const cases = [testcase("portable")]; + const baseline = await fixture.report("baseline.xml", cases); + const directory = join(fixture.root, "reports with spaces"); + await mkdir(directory); + await fixture.report("reports with spaces/shard-a.xml", cases); + await fixture.report("reports with spaces/.hidden.xml", [ + testcase("hidden"), + ]); + await mkdir(join(directory, "nested")); + await fixture.report("reports with spaces/nested/shard-b.xml", [ + testcase("nested"), + ]); + for (const pattern of [ + "reports with spaces/shard-?.xml", + "reports with spaces/shard-[ab].xml", + "reports with spaces/shard-[!b].xml", + "reports with spaces/*.xml", + "**/shard-*.xml", + ]) { + const result = await compare(baseline, join(fixture.root, pattern)); + expect(result.status, `${pattern}: ${result.stderr}`).toBe(0); + } + const hidden = await fixture.report("hidden-baseline.xml", [ + testcase("hidden"), + ]); + expect((await compare(hidden, join(directory, ".*.xml"))).status).toBe(0); + for (const name of ["{a,b}.xml", "!report.xml", "@(shard).xml"]) { + const path = await fixture.report(name, cases); + expect((await compare(baseline, path)).status, name).toBe(0); + } + }); + + test("detects error status and nested summary failures with failure overriding skipped", async () => { + const fixture = await fixtures(); + const baseline = await fixture.report("baseline.xml", [ + testcase("example"), + ]); + for (const status of ["", ""]) { + const candidate = await fixture.report("error.xml", [ + testcase("example", status), + ]); + const result = await compare(baseline, candidate); + expect(result.status).toBe(1); + expect(result.stderr).toContain("test run failed"); + expect(result.stderr).toContain("example > example > failed"); + expect(result.stderr).not.toContain("example > example > skipped"); + } + const nested = join(fixture.root, "nested.xml"); + await writeFile( + nested, + `${testcase("example")}`, + ); + const result = await compare(baseline, nested); + expect(result.status).toBe(1); + expect(result.stderr).toContain("test run failed"); + }); + + test("rejects malformed XML and invalid report numbers", async () => { + const fixture = await fixtures(); + const baseline = await fixture.report("baseline.xml", [ + testcase("example"), + ]); + for (const xml of [ + "", + "", + '', + '', + "", + '', + '', + '', + ]) { + const candidate = join(fixture.root, "invalid.xml"); + await writeFile(candidate, xml); + const result = await compare(baseline, candidate); + expect(result.status, xml).toBe(1); + expect(result.stdout).not.toContain("Identical test inventory"); + } + }); + + test("uses count and timing defaults and only direct unqualified status children", async () => { + const fixture = await fixtures(); + const baseline = await fixture.report("baseline.xml", [ + testcase("example"), + ]); + const candidate = join(fixture.root, "defaults.xml"); + await writeFile( + candidate, + `${testcase("example", "")}`, + ); + const result = await compare(baseline, candidate); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("| defaults.xml | 1 | 0 | 0.00 |"); }); test("rejects ambiguous test identities even when totals match", async () => { diff --git a/sdk/typescript/tsconfig.ci.json b/sdk/typescript/tsconfig.ci.json new file mode 100644 index 000000000..527df088e --- /dev/null +++ b/sdk/typescript/tsconfig.ci.json @@ -0,0 +1,17 @@ +{ + "extends": "./tsconfig.json", + "include": [ + "../../.github/scripts/*.mts", + "scripts/compare-test-reports.mts" + ], + "compilerOptions": { + "rootDir": "../..", + "noEmit": false, + "noEmitOnError": true, + "incremental": false, + "module": "NodeNext", + "moduleResolution": "NodeNext", + "target": "ES2022", + "types": ["node"] + } +} diff --git a/sdk/typescript/tsconfig.json b/sdk/typescript/tsconfig.json index 18454c78e..6738b4dcf 100644 --- a/sdk/typescript/tsconfig.json +++ b/sdk/typescript/tsconfig.json @@ -5,6 +5,8 @@ "dashboard/**/*.ts", "dashboard/**/*.tsx", "tests-ts/**/*.ts", + "../../.github/scripts/*.mts", + "scripts/compare-test-reports.mts", "scripts/smoke-findings-service.ts", "scripts/fixtures/findings-service-sqlite.ts", "scripts/fixtures/prepare-runner-scan.ts"