diff --git a/docs/concepts.md b/docs/concepts.md index b602ae6..22d5146 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -8,7 +8,7 @@ Python coverage is experimental and degraded-honest. Graph-backed structure, unt ## Findings -Findings are named signals such as `typescript.type_errors`, `rust.source_hygiene`, `python.syntax`, `python.source-hygiene`, `python.types`, `python.import-graph`, `python.dead-code`, `python.relevant-tests`, and `coverage.unsupported_stacks`. Counts come from validation diagnostics, graph evidence when available, optional-tool degradation, and repository census data. Opcore does not blend them into a single rating. +Findings are named signals such as `typescript.type_errors`, `rust.source_hygiene`, `python.syntax`, `python.source-hygiene`, `python.types`, `python.import-graph`, `python.dead-code`, `python.relevant-tests`, `python.pytest`, and `coverage.unsupported_stacks`. Counts come from validation diagnostics, graph evidence when available, optional-tool degradation, and repository census data. Opcore does not blend them into a single rating. ## Artifacts diff --git a/packages/asp-provider/src/mapping.ts b/packages/asp-provider/src/mapping.ts index e22255c..48cf5c0 100644 --- a/packages/asp-provider/src/mapping.ts +++ b/packages/asp-provider/src/mapping.ts @@ -684,6 +684,33 @@ function validationEvidence(result: ValidationResult, selectedIds: readonly stri }); continue; } + if (run.capability === "pytest") { + evidence.push({ + kind: "python_validation_capability_run", + message: `Python pytest capability evidence for ${run.projectRoot ?? run.checkId}.`, + data: { + capability: run.capability, + checkId: run.checkId, + activation: run.activation, + outcome: run.outcome, + message: run.message, + ...(run.projectKey === undefined ? {} : { projectKey: run.projectKey }), + ...(run.projectRoot === undefined ? {} : { projectRoot: run.projectRoot }), + ...(run.configFile === undefined ? {} : { configFile: run.configFile }), + ...(run.afterStateFingerprint === undefined ? {} : { afterStateFingerprint: run.afterStateFingerprint }), + ...(run.targetCount === undefined ? {} : { targetCount: run.targetCount }), + ...(run.candidatePaths === undefined ? {} : { candidatePaths: [...run.candidatePaths] }), + ...(run.collectedNodeIds === undefined ? {} : { collectedNodeIds: [...run.collectedNodeIds] }), + ...(run.selectionMode === undefined ? {} : { selectionMode: run.selectionMode }), + ...(run.selectionDigest === undefined ? {} : { selectionDigest: run.selectionDigest }), + ...(run.counts === undefined ? {} : { counts: { ...run.counts } }), + ...(run.collection === undefined ? {} : { collection: { ...run.collection } }), + ...(run.execution === undefined ? {} : { execution: { ...run.execution } }), + ...(run.cleanup === undefined ? {} : { cleanup: { ...run.cleanup } }) + } + }); + continue; + } evidence.push({ kind: "python_validation_capability_run", message: `Python ${run.capability} capability evidence for ${run.checkId}.`, diff --git a/packages/contracts/schemas/opcore-contracts.schema.json b/packages/contracts/schemas/opcore-contracts.schema.json index 232cd7d..4aefbac 100644 --- a/packages/contracts/schemas/opcore-contracts.schema.json +++ b/packages/contracts/schemas/opcore-contracts.schema.json @@ -3371,6 +3371,10 @@ "type": "array", "items": { "$ref": "#/$defs/PythonProjectContext" } }, + "pythonCapabilityRuns": { + "type": "array", + "items": { "$ref": "#/$defs/PythonValidationCapabilityRun" } + }, "manifest": { "type": "object", "additionalProperties": false, diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 8d89f6f..027509b 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -186,7 +186,7 @@ export type PythonValidationAuthoritySource = (typeof pythonValidationAuthorityS export const pythonValidationCapabilityTerminationKinds = ["exited", "timeout", "signal", "spawn_error"] as const; export type PythonValidationCapabilityTerminationKind = (typeof pythonValidationCapabilityTerminationKinds)[number]; -export const pythonValidationCapabilities = ["types", "ruff_lint", "ruff_format"] as const; +export const pythonValidationCapabilities = ["types", "ruff_lint", "ruff_format", "pytest"] as const; export type PythonValidationCapability = (typeof pythonValidationCapabilities)[number]; export const pythonValidationCapabilityStates = [ @@ -1409,7 +1409,8 @@ export interface PythonRuffValidationCapabilityRun { export type PythonValidationCapabilityRun = | PythonTypesValidationCapabilityRun - | PythonRuffValidationCapabilityRun; + | PythonRuffValidationCapabilityRun + | PythonPytestValidationCapabilityRun; export interface PythonValidationCapabilityInvocation { argv: readonly string[]; @@ -1445,6 +1446,70 @@ export interface ValidationResultManifest { skippedChecks?: readonly ValidationSkippedCheck[]; } +export const pythonCapabilityActivations = ["enabled", "disabled", "not_applicable"] as const; +export type PythonCapabilityActivation = (typeof pythonCapabilityActivations)[number]; + +export const pythonPytestSelectionModes = ["none", "direct_argv", "manifest"] as const; +export type PythonPytestSelectionMode = (typeof pythonPytestSelectionModes)[number]; + +export const pythonCapabilityProcessTerminations = ["exited", "timeout", "signal", "spawn_error", "overflow"] as const; +export type PythonCapabilityProcessTermination = (typeof pythonCapabilityProcessTerminations)[number]; + +export interface PythonCapabilityCounts { + candidateCount: number; + collectedCount: number; + executedCount: number; + passedCount: number; + failedCount: number; + skippedCount: number; + xfailedCount: number; + xpassedCount: number; + errorCount: number; +} + +export interface PythonCapabilityCleanupEvidence { + attempted: boolean; + ok: boolean; + failureMessage?: string; +} + +export interface PythonCapabilityInvocation { + stage: "collection" | "execution"; + command: string; + argsDigest: string; + argCount: number; + selectionMode: PythonPytestSelectionMode; + selectionDigest?: string; + durationMs: number; + termination: PythonCapabilityProcessTermination; + exitCode?: number; + signal?: string; + outputBytes: number; + stdoutDigest?: string; + stderrDigest?: string; +} + +export interface PythonPytestValidationCapabilityRun { + capability: "pytest"; + checkId: string; + activation: PythonCapabilityActivation; + outcome: string; + message: string; + projectKey?: string; + projectRoot?: string; + configFile?: string; + targetCount?: number; + candidatePaths?: readonly string[]; + collectedNodeIds?: readonly string[]; + afterStateFingerprint?: string; + selectionMode?: PythonPytestSelectionMode; + selectionDigest?: string; + counts?: PythonCapabilityCounts; + collection?: PythonCapabilityInvocation; + execution?: PythonCapabilityInvocation; + cleanup?: PythonCapabilityCleanupEvidence; +} + export interface ValidationResult { ok: boolean; status: ValidationResultStatus; @@ -6744,9 +6809,9 @@ export function validateValidationResultPayload(result: ValidationResult): Valid export function validatePythonValidationCapabilityRun( run: PythonValidationCapabilityRun ): PythonValidationCapabilityRun { - return run.capability === "types" - ? validatePythonTypesValidationCapabilityRun(run) - : validatePythonRuffValidationCapabilityRun(run); + if (run.capability === "types") return validatePythonTypesValidationCapabilityRun(run); + if (run.capability === "pytest") return validatePythonPytestValidationCapabilityRun(run); + return validatePythonRuffValidationCapabilityRun(run); } function validatePythonTypesValidationCapabilityRun( @@ -6891,6 +6956,130 @@ export function validatePythonValidationCapabilityRuns( return runs; } +function validatePythonPytestValidationCapabilityRun( + run: PythonPytestValidationCapabilityRun +): PythonPytestValidationCapabilityRun { + if (!run || typeof run !== "object") throw new Error("Python pytest capability run is required"); + validateExactObjectKeys(run, [ + "capability", + "checkId", + "activation", + "outcome", + "message", + "projectKey", + "projectRoot", + "configFile", + "targetCount", + "candidatePaths", + "collectedNodeIds", + "afterStateFingerprint", + "selectionMode", + "selectionDigest", + "counts", + "collection", + "execution", + "cleanup" + ], "Python pytest capability run"); + if (run.capability !== "pytest" || run.checkId !== "python.pytest") { + throw new Error("Python pytest capability run must describe python.pytest"); + } + if (!includesString(pythonCapabilityActivations, run.activation)) { + throw new Error(`Unknown Python pytest capability activation: ${String(run.activation)}`); + } + validateNonEmptyString(run.outcome, "Python pytest capability run outcome"); + validateNonEmptyString(run.message, "Python pytest capability run message"); + if (run.projectKey !== undefined) validateSha256Identity(run.projectKey, "Python pytest capability run projectKey"); + if (run.projectRoot !== undefined) validatePythonProjectRoot(run.projectRoot, "Python pytest capability run projectRoot"); + if (run.configFile !== undefined) validateRepoRelativePath(run.configFile); + if (run.targetCount !== undefined) validateNonNegativeInteger(run.targetCount, "Python pytest capability run targetCount"); + if (run.candidatePaths !== undefined) validateRepoPathArray(run.candidatePaths, "Python pytest capability run candidatePaths"); + if (run.collectedNodeIds !== undefined) { + validateStringArray(run.collectedNodeIds, "Python pytest capability run collectedNodeIds", { allowEmpty: true }); + } + if (run.afterStateFingerprint !== undefined) { + validateSha256Identity(run.afterStateFingerprint, "Python pytest capability run afterStateFingerprint"); + } + if (run.selectionMode !== undefined && !includesString(pythonPytestSelectionModes, run.selectionMode)) { + throw new Error(`Unknown Python pytest capability selection mode: ${String(run.selectionMode)}`); + } + if (run.selectionDigest !== undefined) { + validateSha256Identity(run.selectionDigest, "Python pytest capability run selectionDigest"); + } + if (run.counts !== undefined) validatePythonPytestCapabilityCounts(run.counts); + if (run.collection !== undefined) validatePythonPytestCapabilityInvocation(run.collection, "Python pytest capability collection"); + if (run.execution !== undefined) validatePythonPytestCapabilityInvocation(run.execution, "Python pytest capability execution"); + if (run.cleanup !== undefined) validatePythonPytestCapabilityCleanupEvidence(run.cleanup); + return run; +} + +function validatePythonPytestCapabilityCounts(counts: PythonCapabilityCounts): void { + if (!counts || typeof counts !== "object") throw new Error("Python pytest capability counts are required"); + validateExactObjectKeys(counts, [ + "candidateCount", + "collectedCount", + "executedCount", + "passedCount", + "failedCount", + "skippedCount", + "xfailedCount", + "xpassedCount", + "errorCount" + ], "Python pytest capability counts"); + for (const key of Object.keys(counts) as (keyof PythonCapabilityCounts)[]) { + validateNonNegativeInteger(counts[key], `Python pytest capability counts ${key}`); + } +} + +function validatePythonPytestCapabilityInvocation(invocation: PythonCapabilityInvocation, label: string): void { + if (!invocation || typeof invocation !== "object") throw new Error(`${label} is required`); + validateExactObjectKeys(invocation, [ + "stage", + "command", + "argsDigest", + "argCount", + "selectionMode", + "selectionDigest", + "durationMs", + "termination", + "exitCode", + "signal", + "outputBytes", + "stdoutDigest", + "stderrDigest" + ], label); + if (!includesString(["collection", "execution"] as const, invocation.stage)) { + throw new Error(`${label} stage must be collection or execution`); + } + validateNonEmptyString(invocation.command, `${label} command`); + validateSha256Identity(invocation.argsDigest, `${label} argsDigest`); + validateNonNegativeInteger(invocation.argCount, `${label} argCount`); + if (!includesString(pythonPytestSelectionModes, invocation.selectionMode)) { + throw new Error(`Unknown ${label} selectionMode: ${String(invocation.selectionMode)}`); + } + if (invocation.selectionDigest !== undefined) { + validateSha256Identity(invocation.selectionDigest, `${label} selectionDigest`); + } + validateNonNegativeNumber(invocation.durationMs, `${label} durationMs`); + if (!includesString(pythonCapabilityProcessTerminations, invocation.termination)) { + throw new Error(`Unknown ${label} termination: ${String(invocation.termination)}`); + } + if (invocation.exitCode !== undefined) validateNonNegativeInteger(invocation.exitCode, `${label} exitCode`); + if (invocation.signal !== undefined) validateNonEmptyString(invocation.signal, `${label} signal`); + validateNonNegativeInteger(invocation.outputBytes, `${label} outputBytes`); + if (invocation.stdoutDigest !== undefined) validateSha256Identity(invocation.stdoutDigest, `${label} stdoutDigest`); + if (invocation.stderrDigest !== undefined) validateSha256Identity(invocation.stderrDigest, `${label} stderrDigest`); +} + +function validatePythonPytestCapabilityCleanupEvidence(cleanup: PythonCapabilityCleanupEvidence): void { + if (!cleanup || typeof cleanup !== "object") throw new Error("Python pytest capability cleanup evidence is required"); + validateExactObjectKeys(cleanup, ["attempted", "ok", "failureMessage"], "Python pytest capability cleanup evidence"); + if (typeof cleanup.attempted !== "boolean") throw new Error("Python pytest capability cleanup attempted must be boolean"); + if (typeof cleanup.ok !== "boolean") throw new Error("Python pytest capability cleanup ok must be boolean"); + if (cleanup.failureMessage !== undefined) { + validateNonEmptyString(cleanup.failureMessage, "Python pytest capability cleanup failureMessage"); + } +} + function validatePythonValidationCapabilityTool( tool: PythonValidationCapabilityToolProvenance, run: PythonTypesValidationCapabilityRun @@ -7099,6 +7288,72 @@ export function validatePythonProjectContexts(contexts: readonly PythonProjectCo return contexts; } +function validatePythonCapabilityCounts(counts: PythonCapabilityCounts): void { + if (!counts || typeof counts !== "object") throw new Error("Python capability counts are required"); + validateExactObjectKeys(counts, [ + "candidateCount", + "collectedCount", + "executedCount", + "passedCount", + "failedCount", + "skippedCount", + "xfailedCount", + "xpassedCount", + "errorCount" + ], "Python capability counts"); + for (const key of Object.keys(counts) as (keyof PythonCapabilityCounts)[]) { + validateNonNegativeInteger(counts[key], `Python capability counts ${key}`); + } +} + +function validatePythonCapabilityCleanupEvidence(cleanup: PythonCapabilityCleanupEvidence): void { + if (!cleanup || typeof cleanup !== "object") throw new Error("Python capability cleanup evidence is required"); + validateExactObjectKeys(cleanup, ["attempted", "ok", "failureMessage"], "Python capability cleanup evidence"); + if (typeof cleanup.attempted !== "boolean") throw new Error("Python capability cleanup attempted must be boolean"); + if (typeof cleanup.ok !== "boolean") throw new Error("Python capability cleanup ok must be boolean"); + if (cleanup.failureMessage !== undefined) { + validateNonEmptyString(cleanup.failureMessage, "Python capability cleanup failureMessage"); + } +} + +function validatePythonCapabilityInvocation(invocation: PythonCapabilityInvocation, label: string): void { + if (!invocation || typeof invocation !== "object") throw new Error(`${label} is required`); + validateExactObjectKeys(invocation, [ + "stage", + "command", + "argsDigest", + "argCount", + "selectionMode", + "selectionDigest", + "durationMs", + "termination", + "exitCode", + "signal", + "outputBytes", + "stdoutDigest", + "stderrDigest" + ], label); + if (!includesString(["collection", "execution"] as const, invocation.stage)) { + throw new Error(`${label} stage must be collection or execution`); + } + validateNonEmptyString(invocation.command, `${label} command`); + validateSha256Identity(invocation.argsDigest, `${label} argsDigest`); + validateNonNegativeInteger(invocation.argCount, `${label} argCount`); + if (!includesString(pythonPytestSelectionModes, invocation.selectionMode)) { + throw new Error(`Unknown ${label} selectionMode: ${String(invocation.selectionMode)}`); + } + if (invocation.selectionDigest !== undefined) validateSha256Identity(invocation.selectionDigest, `${label} selectionDigest`); + validateNonNegativeInteger(invocation.durationMs, `${label} durationMs`); + if (!includesString(pythonCapabilityProcessTerminations, invocation.termination)) { + throw new Error(`Unknown ${label} termination: ${String(invocation.termination)}`); + } + if (invocation.exitCode !== undefined) validateNonNegativeInteger(invocation.exitCode, `${label} exitCode`); + if (invocation.signal !== undefined) validateNonEmptyString(invocation.signal, `${label} signal`); + validateNonNegativeInteger(invocation.outputBytes, `${label} outputBytes`); + if (invocation.stdoutDigest !== undefined) validateSha256Identity(invocation.stdoutDigest, `${label} stdoutDigest`); + if (invocation.stderrDigest !== undefined) validateSha256Identity(invocation.stderrDigest, `${label} stderrDigest`); +} + function validatePythonProjectTarget(target: PythonProjectTarget): void { if (!target || typeof target !== "object") throw new Error("Python project targetRuntime is required"); validateExactObjectKeys( @@ -7732,8 +7987,8 @@ function validatePythonRuffValidationCapabilityRun( } if (run.schemaVersion !== 1) throw new Error("Python validation capability run schemaVersion must be 1"); validateValidationCheckId(run.checkId, "Python validation capability run checkId"); - if (!includesString(pythonValidationCapabilities, run.capability)) { - throw new Error(`Unknown Python validation capability: ${String(run.capability)}`); + if (!includesString(["ruff_lint", "ruff_format"] as const, run.capability)) { + throw new Error(`Unknown Python Ruff validation capability: ${String(run.capability)}`); } if (!includesString(pythonValidationCapabilityStates, run.state)) { throw new Error(`Unknown Python validation capability state: ${String(run.state)}`); diff --git a/packages/fixtures/src/index.ts b/packages/fixtures/src/index.ts index 1c524b4..c495555 100644 --- a/packages/fixtures/src/index.ts +++ b/packages/fixtures/src/index.ts @@ -611,7 +611,8 @@ export const conformanceFixtureMetadata = [ "python.types", "python.import-graph", "python.dead-code", - "python.relevant-tests" + "python.relevant-tests", + "python.pytest" ], degradedTools: ["mypy", "pyright", "ruff", "pytest"] } diff --git a/packages/fixtures/validation-python/README.md b/packages/fixtures/validation-python/README.md index f73c6f6..d62d24f 100644 --- a/packages/fixtures/validation-python/README.md +++ b/packages/fixtures/validation-python/README.md @@ -3,3 +3,5 @@ `mypy-authority/` is the real-tool proof repository for `python.types`. It combines strict mypy configuration, a configured plugin, a stub-only dependency, and a namespace package. Source and configuration resources retain a `.fixture` suffix on disk so Opcore does not discover them as live repository inputs; the proof script strips that suffix in its isolated workspace. `scripts/check-python-mypy-authority.mjs` runs pinned mypy over both clean and hypothetical/materialized after-states and verifies portable manifest identity plus cleanup. `pyright-authority/` is the real-tool Pyright proof repository. It covers JSONC, recursive extends, include/exclude/ignore, extra paths, strict execution environments, namespace/src/stub layouts, hypothetical/materialized equivalence, portable receipts, packed Opcore execution, and cleanup with pinned Pyright 1.1.411. + +`pytest-authority/` contains opt-in real-tool pytest authority fixtures for pass, failure, collection failure, timeout, manifest fallback, ambient-plugin isolation, and workspace isolation. Synthetic source, test, and packaging files retain a `.fixture` suffix so repository validation and provenance checks do not treat them as live product source; the authority proof materializes canonical filenames in an isolated workspace. diff --git a/packages/fixtures/validation-python/pytest-authority/collection-fail/pyproject.toml.fixture b/packages/fixtures/validation-python/pytest-authority/collection-fail/pyproject.toml.fixture new file mode 100644 index 0000000..35c01ab --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/collection-fail/pyproject.toml.fixture @@ -0,0 +1,3 @@ +[project] +name = "pytest-authority-collection-fail" +version = "0.0.0" diff --git a/packages/fixtures/validation-python/pytest-authority/collection-fail/src/app.py.fixture b/packages/fixtures/validation-python/pytest-authority/collection-fail/src/app.py.fixture new file mode 100644 index 0000000..4860f40 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/collection-fail/src/app.py.fixture @@ -0,0 +1,2 @@ +def answer() -> int: + return 42 diff --git a/packages/fixtures/validation-python/pytest-authority/collection-fail/tests/conftest.py.fixture b/packages/fixtures/validation-python/pytest-authority/collection-fail/tests/conftest.py.fixture new file mode 100644 index 0000000..5801bea --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/collection-fail/tests/conftest.py.fixture @@ -0,0 +1,4 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) diff --git a/packages/fixtures/validation-python/pytest-authority/collection-fail/tests/test_app.py.fixture b/packages/fixtures/validation-python/pytest-authority/collection-fail/tests/test_app.py.fixture new file mode 100644 index 0000000..837d47e --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/collection-fail/tests/test_app.py.fixture @@ -0,0 +1,6 @@ +from src.app import answer +from missing_fixture_dependency import NEVER_DEFINED + + +def test_answer() -> None: + assert answer() == NEVER_DEFINED diff --git a/packages/fixtures/validation-python/pytest-authority/fail/pyproject.toml.fixture b/packages/fixtures/validation-python/pytest-authority/fail/pyproject.toml.fixture new file mode 100644 index 0000000..ff678d3 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/fail/pyproject.toml.fixture @@ -0,0 +1,3 @@ +[project] +name = "pytest-authority-fail" +version = "0.0.0" diff --git a/packages/fixtures/validation-python/pytest-authority/fail/src/app.py.fixture b/packages/fixtures/validation-python/pytest-authority/fail/src/app.py.fixture new file mode 100644 index 0000000..ffa919e --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/fail/src/app.py.fixture @@ -0,0 +1,3 @@ +def answer() -> int: + return 42 + diff --git a/packages/fixtures/validation-python/pytest-authority/fail/tests/conftest.py.fixture b/packages/fixtures/validation-python/pytest-authority/fail/tests/conftest.py.fixture new file mode 100644 index 0000000..6ee195b --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/fail/tests/conftest.py.fixture @@ -0,0 +1,5 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + diff --git a/packages/fixtures/validation-python/pytest-authority/fail/tests/test_app.py.fixture b/packages/fixtures/validation-python/pytest-authority/fail/tests/test_app.py.fixture new file mode 100644 index 0000000..7291019 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/fail/tests/test_app.py.fixture @@ -0,0 +1,5 @@ +from src.app import answer + + +def test_answer() -> None: + assert answer() == 0 diff --git a/packages/fixtures/validation-python/pytest-authority/manifest/pyproject.toml.fixture b/packages/fixtures/validation-python/pytest-authority/manifest/pyproject.toml.fixture new file mode 100644 index 0000000..e9ac466 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/manifest/pyproject.toml.fixture @@ -0,0 +1,3 @@ +[project] +name = "pytest-authority-manifest" +version = "0.0.0" diff --git a/packages/fixtures/validation-python/pytest-authority/manifest/src/app.py.fixture b/packages/fixtures/validation-python/pytest-authority/manifest/src/app.py.fixture new file mode 100644 index 0000000..4860f40 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/manifest/src/app.py.fixture @@ -0,0 +1,2 @@ +def answer() -> int: + return 42 diff --git a/packages/fixtures/validation-python/pytest-authority/manifest/tests/conftest.py.fixture b/packages/fixtures/validation-python/pytest-authority/manifest/tests/conftest.py.fixture new file mode 100644 index 0000000..5801bea --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/manifest/tests/conftest.py.fixture @@ -0,0 +1,4 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) diff --git a/packages/fixtures/validation-python/pytest-authority/manifest/tests/test_app.py.fixture b/packages/fixtures/validation-python/pytest-authority/manifest/tests/test_app.py.fixture new file mode 100644 index 0000000..455177c --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/manifest/tests/test_app.py.fixture @@ -0,0 +1,11 @@ +import pytest + +from src.app import answer + +VALUES = [f"case-{index:03d}-{'x' * 120}" for index in range(180)] + + +@pytest.mark.parametrize("label", VALUES, ids=VALUES) +def test_answer(label: str) -> None: + assert label.startswith("case-") + assert answer() == 42 diff --git a/packages/fixtures/validation-python/pytest-authority/pass/pyproject.toml.fixture b/packages/fixtures/validation-python/pytest-authority/pass/pyproject.toml.fixture new file mode 100644 index 0000000..5bc5e17 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/pass/pyproject.toml.fixture @@ -0,0 +1,3 @@ +[project] +name = "pytest-authority-pass" +version = "0.0.0" diff --git a/packages/fixtures/validation-python/pytest-authority/pass/src/app.py.fixture b/packages/fixtures/validation-python/pytest-authority/pass/src/app.py.fixture new file mode 100644 index 0000000..ffa919e --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/pass/src/app.py.fixture @@ -0,0 +1,3 @@ +def answer() -> int: + return 42 + diff --git a/packages/fixtures/validation-python/pytest-authority/pass/tests/conftest.py.fixture b/packages/fixtures/validation-python/pytest-authority/pass/tests/conftest.py.fixture new file mode 100644 index 0000000..6ee195b --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/pass/tests/conftest.py.fixture @@ -0,0 +1,5 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + diff --git a/packages/fixtures/validation-python/pytest-authority/pass/tests/test_app.py.fixture b/packages/fixtures/validation-python/pytest-authority/pass/tests/test_app.py.fixture new file mode 100644 index 0000000..ca059ed --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/pass/tests/test_app.py.fixture @@ -0,0 +1,6 @@ +from src.app import answer + + +def test_answer() -> None: + assert answer() == 42 + diff --git a/packages/fixtures/validation-python/pytest-authority/timeout/pyproject.toml.fixture b/packages/fixtures/validation-python/pytest-authority/timeout/pyproject.toml.fixture new file mode 100644 index 0000000..a0a2e34 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/timeout/pyproject.toml.fixture @@ -0,0 +1,3 @@ +[project] +name = "pytest-authority-timeout" +version = "0.0.0" diff --git a/packages/fixtures/validation-python/pytest-authority/timeout/src/app.py.fixture b/packages/fixtures/validation-python/pytest-authority/timeout/src/app.py.fixture new file mode 100644 index 0000000..4860f40 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/timeout/src/app.py.fixture @@ -0,0 +1,2 @@ +def answer() -> int: + return 42 diff --git a/packages/fixtures/validation-python/pytest-authority/timeout/tests/conftest.py.fixture b/packages/fixtures/validation-python/pytest-authority/timeout/tests/conftest.py.fixture new file mode 100644 index 0000000..5801bea --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/timeout/tests/conftest.py.fixture @@ -0,0 +1,4 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) diff --git a/packages/fixtures/validation-python/pytest-authority/timeout/tests/test_app.py.fixture b/packages/fixtures/validation-python/pytest-authority/timeout/tests/test_app.py.fixture new file mode 100644 index 0000000..3fa8fcc --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/timeout/tests/test_app.py.fixture @@ -0,0 +1,8 @@ +import time + +from src.app import answer + + +def test_answer() -> None: + time.sleep(31) + assert answer() == 42 diff --git a/packages/fixtures/validation-python/pytest-authority/workspace-isolation/pyproject.toml.fixture b/packages/fixtures/validation-python/pytest-authority/workspace-isolation/pyproject.toml.fixture new file mode 100644 index 0000000..bd362b6 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/workspace-isolation/pyproject.toml.fixture @@ -0,0 +1,3 @@ +[project] +name = "pytest-authority-workspace-isolation" +version = "0.0.0" diff --git a/packages/fixtures/validation-python/pytest-authority/workspace-isolation/src/app.py.fixture b/packages/fixtures/validation-python/pytest-authority/workspace-isolation/src/app.py.fixture new file mode 100644 index 0000000..4860f40 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/workspace-isolation/src/app.py.fixture @@ -0,0 +1,2 @@ +def answer() -> int: + return 42 diff --git a/packages/fixtures/validation-python/pytest-authority/workspace-isolation/tests/conftest.py.fixture b/packages/fixtures/validation-python/pytest-authority/workspace-isolation/tests/conftest.py.fixture new file mode 100644 index 0000000..b3a5741 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/workspace-isolation/tests/conftest.py.fixture @@ -0,0 +1,12 @@ +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + + +def pytest_collection_modifyitems(session, config, items): + (Path(config.rootpath) / "src" / "app.py").write_text( + "def answer() -> int:\n return 0\n", + encoding="utf-8" + ) diff --git a/packages/fixtures/validation-python/pytest-authority/workspace-isolation/tests/test_app.py.fixture b/packages/fixtures/validation-python/pytest-authority/workspace-isolation/tests/test_app.py.fixture new file mode 100644 index 0000000..913d39a --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/workspace-isolation/tests/test_app.py.fixture @@ -0,0 +1,5 @@ +from src.app import answer + + +def test_answer() -> None: + assert answer() == 42 diff --git a/packages/opcore/src/init.ts b/packages/opcore/src/init.ts index 0674bd2..63ce789 100644 --- a/packages/opcore/src/init.ts +++ b/packages/opcore/src/init.ts @@ -755,7 +755,8 @@ function checksForLanguage(language: string, validation: OpcoreInitLanguageSetti "python.types", "python.import-graph", "python.dead-code", - "python.relevant-tests" + "python.relevant-tests", + "python.pytest" ]; } return []; diff --git a/packages/opcore/src/reporting.ts b/packages/opcore/src/reporting.ts index 5a31bf8..045c20e 100644 --- a/packages/opcore/src/reporting.ts +++ b/packages/opcore/src/reporting.ts @@ -232,11 +232,11 @@ export function createOpcoreMetricReport(input: CreateOpcoreMetricReportInput): }); addDiagnosticSignal(signals, { id: "python.untested_modules", - title: "Python modules without TESTED_BY graph evidence", + title: "Python modules without TESTED_BY graph candidate evidence", category: "python", severity: "warning", checkId: pythonRelevantTestsCheckId, - diagnostics: diagnostics.filter((diagnostic) => diagnostic.code === "PY_RELEVANT_TESTS_ABSENT" && hasPath(diagnostic)) + diagnostics: diagnostics.filter((diagnostic) => diagnostic.code === "PY_RELEVANT_TEST_CANDIDATES_ABSENT" && hasPath(diagnostic)) }); addReceiptBackedDiagnosticSignal(signals, validationResult, { id: "python.ruff_lint_findings", @@ -808,7 +808,7 @@ function isProvenRuffFindingsRun( run: PythonValidationCapabilityRun ): run is PythonRuffValidationCapabilityRun { const sha256Identity = /^sha256:[a-f0-9]{64}$/u; - return run.capability !== "types" && + return (run.capability === "ruff_lint" || run.capability === "ruff_format") && run.state === "findings" && run.checkId === (run.capability === "ruff_lint" ? "python.ruff-lint" : "python.ruff-format") && typeof run.projectKey === "string" && diff --git a/packages/validation-policy/src/factory.ts b/packages/validation-policy/src/factory.ts index 806fad7..0423758 100644 --- a/packages/validation-policy/src/factory.ts +++ b/packages/validation-policy/src/factory.ts @@ -3,7 +3,7 @@ import type { ValidationCheckDefinition } from "@the-open-engine/opcore-validati import { createValidationCheckRegistry } from "@the-open-engine/opcore-validation"; import { createCloneValidationChecks } from "@the-open-engine/opcore-validation-clone"; import { createDocsValidationChecks, type CreateDocsValidationChecksOptions } from "@the-open-engine/opcore-validation-docs"; -import { createNodePythonProjectWorkspace, createPythonValidationChecks } from "@the-open-engine/opcore-validation-python"; +import { createNodePythonProjectWorkspace, createPythonValidationChecks, disabledPytestRun, PYTHON_PYTEST_CHECK_ID } from "@the-open-engine/opcore-validation-python"; import { createRustValidationChecks } from "@the-open-engine/opcore-validation-rust"; import { createTypeScriptValidationChecks } from "@the-open-engine/opcore-validation-typescript"; import { readOpcoreRepoConfig } from "./config.js"; @@ -98,6 +98,7 @@ function applyValidationPolicy( const filteredContexts = new WeakMap[0]>(); const checks = available .filter((check) => adapters === undefined || adapters.has(check.adapter)) + .map((check) => applyDefaultScopePolicy(check, defaults)) .map((check) => applyDisabledPolicy(check, disabled)) .filter((check): check is ValidationCheckDefinition => check !== undefined) @@ -175,6 +176,9 @@ function applyDisabledPolicy( return { ...check, defaultScopes: [], + requiresGraph: false, + graphUsage: "none", + graphRequirements: undefined, inactiveStateWhenUnselected: "disabled", run: (context) => check.inactiveResult?.(context, "disabled") }; diff --git a/packages/validation-python/src/check-ids.ts b/packages/validation-python/src/check-ids.ts index dd59c78..d2553c2 100644 --- a/packages/validation-python/src/check-ids.ts +++ b/packages/validation-python/src/check-ids.ts @@ -6,6 +6,7 @@ export const PYTHON_TYPES_CHECK_ID = "python.types"; export const PYTHON_IMPORT_GRAPH_CHECK_ID = "python.import-graph"; export const PYTHON_DEAD_CODE_CHECK_ID = "python.dead-code"; export const PYTHON_RELEVANT_TESTS_CHECK_ID = "python.relevant-tests"; +export const PYTHON_PYTEST_CHECK_ID = "python.pytest"; export const pythonValidationCheckIds = [ PYTHON_SYNTAX_CHECK_ID, @@ -15,7 +16,8 @@ export const pythonValidationCheckIds = [ PYTHON_TYPES_CHECK_ID, PYTHON_IMPORT_GRAPH_CHECK_ID, PYTHON_DEAD_CODE_CHECK_ID, - PYTHON_RELEVANT_TESTS_CHECK_ID + PYTHON_RELEVANT_TESTS_CHECK_ID, + PYTHON_PYTEST_CHECK_ID ] as const; export type PythonValidationCheckId = (typeof pythonValidationCheckIds)[number]; diff --git a/packages/validation-python/src/environment-resolution.ts b/packages/validation-python/src/environment-resolution.ts index c53b20c..aaef28d 100644 --- a/packages/validation-python/src/environment-resolution.ts +++ b/packages/validation-python/src/environment-resolution.ts @@ -235,7 +235,9 @@ async function resolveTools( const prefix = candidate.argv.slice(1); const probePrefix = tool === "ruff" ? withoutToolConfigOptions(prefix, tool) : prefix; const result = await processProbe.run(command, [...probePrefix, "--version"], { - cwd: projectCwd(options), env, timeoutMs: options.timeoutMs ?? 10000 + cwd: projectCwd(options), + env: tool === "pytest" ? { ...env, PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" } : env, + timeoutMs: options.timeoutMs ?? 10000 }); if (!result.ok) { failure = probeFailureReason(result, tool); diff --git a/packages/validation-python/src/index.ts b/packages/validation-python/src/index.ts index 04753ac..302529d 100644 --- a/packages/validation-python/src/index.ts +++ b/packages/validation-python/src/index.ts @@ -8,6 +8,7 @@ import { } from "./check-ids.js"; import { createDeadCodeCheck } from "./dead-code-check.js"; import { createImportGraphCheck } from "./import-graph-check.js"; +import { createPytestCheck } from "./pytest-check.js"; import { createRelevantTestsCheck } from "./relevant-tests-check.js"; import { createRuffFormatCheck, type PythonRuffFormatCheckOptions } from "./ruff-format-check.js"; import { createRuffLintCheck, type PythonRuffLintCheckOptions } from "./ruff-lint-check.js"; @@ -20,6 +21,7 @@ import type { PythonImportAnalyzer } from "./import-analysis.js"; export { PYTHON_DEAD_CODE_CHECK_ID, PYTHON_IMPORT_GRAPH_CHECK_ID, + PYTHON_PYTEST_CHECK_ID, PYTHON_RELEVANT_TESTS_CHECK_ID, PYTHON_RUFF_FORMAT_CHECK_ID, PYTHON_RUFF_LINT_CHECK_ID, @@ -55,6 +57,7 @@ export { } from "./ruff-lint-check.js"; export { createSyntaxCheck, type PythonSyntaxCheckOptions } from "./syntax-check.js"; export { createTypeCheck, type PythonTypeCheckOptions } from "./type-check.js"; +export { disabledPytestRun } from "./pytest-check.js"; export interface CreatePythonValidationChecksOptions extends PythonTypeCheckOptions, PythonSyntaxCheckOptions, PythonRuffLintCheckOptions, PythonRuffFormatCheckOptions { @@ -84,7 +87,8 @@ export function createPythonValidationChecks( createTypeCheck(options, resolveContexts, resolveSources), createImportGraphCheck(resolveSources), createDeadCodeCheck(resolveRoots), - createRelevantTestsCheck(resolveRoots) + createRelevantTestsCheck(resolveRoots), + createPytestCheck(options, resolveContexts) ].map((check) => withPythonProjectContexts(check, resolveContexts)); } diff --git a/packages/validation-python/src/materialized-workspace.ts b/packages/validation-python/src/materialized-workspace.ts new file mode 100644 index 0000000..6480cb1 --- /dev/null +++ b/packages/validation-python/src/materialized-workspace.ts @@ -0,0 +1,56 @@ +import { chmodSync, existsSync, rmSync, writeFileSync } from "node:fs"; +import { mkdir } from "node:fs/promises"; +import { dirname, relative, resolve, sep } from "node:path"; +import { validateRepoRelativePath } from "@the-open-engine/opcore-contracts"; + +export function resolveMaterializedWorkspacePath(root: string, path: string, label: string): string { + const normalized = validateRepoRelativePath(path); + const absolutePath = resolve(root, normalized); + const relativePath = relative(root, absolutePath); + if (relativePath === "" || relativePath.startsWith("..") || relativePath.split(sep).includes("..")) { + throw new Error(`Repo-relative path escapes ${label}: ${path}`); + } + return absolutePath; +} + +export async function writeMaterializedWorkspaceFile( + root: string, + path: string, + content: string, + label: string, + mode?: number +): Promise { + const absolutePath = resolveMaterializedWorkspacePath(root, path, label); + await mkdir(dirname(absolutePath), { recursive: true }); + writeFileSync(absolutePath, content, "utf8"); + if (mode !== undefined) chmodSync(absolutePath, mode & 0o7777); +} + +export function removeMaterializedWorkspace(root: string, label: string): void { + let lastError: unknown; + for (let attempt = 0; attempt < 6; attempt += 1) { + try { + rmSync(root, { recursive: true, force: true }); + if (!existsSync(root)) return; + lastError = new Error(`${label} cleanup left residual files.`); + } catch (error) { + lastError = error; + } + sleep(25); + } + throw sanitizeWorkspaceCleanupError(lastError, label); +} + +function sanitizeWorkspaceCleanupError(error: unknown, label: string): Error { + const code = typeof error === "object" && error !== null && "code" in error && typeof error.code === "string" + ? error.code + : undefined; + const suffix = error instanceof Error && error.message.includes("residual files") + ? "Residual files remained after cleanup retries." + : (code === undefined ? "Cleanup failed." : `Cleanup failed (${code}).`); + return new Error(`${label} ${suffix}`); +} + +function sleep(milliseconds: number): void { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds); +} diff --git a/packages/validation-python/src/node-shims.d.ts b/packages/validation-python/src/node-shims.d.ts index f84946f..1487336 100644 --- a/packages/validation-python/src/node-shims.d.ts +++ b/packages/validation-python/src/node-shims.d.ts @@ -54,18 +54,20 @@ declare module "node:crypto" { } declare module "node:fs" { + export function chmodSync(path: string, mode: number): void; export function existsSync(path: string): boolean; + export function readFileSync(path: string, encoding: "utf8"): string; export function realpathSync(path: string): string; export function mkdirSync(path: string, options?: { recursive?: boolean }): void; export function mkdtempSync(prefix: string): string; export function rmSync(path: string, options?: { recursive?: boolean; force?: boolean }): void; - export function writeFileSync(path: string, data: string): void; + export function writeFileSync(path: string, data: string, encoding?: "utf8"): void; } declare module "node:fs/promises" { export function access(path: string): Promise; export function copyFile(source: string, destination: string): Promise; - export function lstat(path: string): Promise<{ isFile(): boolean; isSymbolicLink(): boolean }>; + export function lstat(path: string): Promise<{ mode: number; isFile(): boolean; isSymbolicLink(): boolean }>; export function mkdir(path: string, options?: { recursive?: boolean }): Promise; export function readFile(path: string, encoding: "utf8"): Promise; export function readdir(path: string, options: { recursive: true }): Promise; @@ -119,6 +121,10 @@ declare namespace NodeJS { } } +declare const Buffer: { + byteLength(input: string, encoding?: BufferEncoding): number; +}; + declare const process: { env: Record; execPath: string; diff --git a/packages/validation-python/src/project-config-files.ts b/packages/validation-python/src/project-config-files.ts index ed16565..1201a5c 100644 --- a/packages/validation-python/src/project-config-files.ts +++ b/packages/validation-python/src/project-config-files.ts @@ -1,5 +1,5 @@ export const pythonBoundaryFileNames = [ - "pyproject.toml", "Pipfile", "Pipfile.lock", "poetry.lock", "pdm.lock", "uv.lock", "setup.cfg", "setup.py" + "pyproject.toml", "Pipfile", "Pipfile.lock", "poetry.lock", "pdm.lock", "uv.lock", "setup.cfg", "setup.py", "pytest.ini", "tox.ini" ] as const; const pythonToolConfigFileNames = [ @@ -9,6 +9,7 @@ const pythonToolConfigFileNames = [ export function isRelevantPythonConfig(path: string): boolean { const name = path.slice(path.lastIndexOf("/") + 1); return pythonBoundaryFileNames.includes(name as (typeof pythonBoundaryFileNames)[number]) || + name === "conftest.py" || /^requirements.*\.txt$/u.test(name) || pythonToolConfigFileNames.includes(name as (typeof pythonToolConfigFileNames)[number]); } diff --git a/packages/validation-python/src/project-workspace.ts b/packages/validation-python/src/project-workspace.ts index 8d801c0..4e80bb0 100644 --- a/packages/validation-python/src/project-workspace.ts +++ b/packages/validation-python/src/project-workspace.ts @@ -14,9 +14,11 @@ export interface PythonProjectWorkspaceRealpath { export interface PythonProjectWorkspace { read(path: string): Promise; list(): Promise; + listAll?(): Promise; exists(path: string): Promise; realpath(path: string): Promise; executableExists(path: string): Promise; + statMode?(path: string): Promise; } export function createValidationFileViewPythonWorkspace( @@ -25,33 +27,34 @@ export function createValidationFileViewPythonWorkspace( fullWorkspace?: PythonProjectWorkspace ): PythonProjectWorkspace { const executableAvailable = executableExists ?? fullWorkspace?.executableExists ?? nodeExecutableExists; + const listPaths = async (include: (path: string) => boolean) => { + const fileViewPaths = await fileView.listVisibleFiles(); + const fullWorkspacePaths = fullWorkspace === undefined ? [] : await fullWorkspace.list(); + const fileViewPathSet = new Set(fileViewPaths); + const fullWorkspacePathSet = new Set(fullWorkspacePaths); + const candidates = [...new Set([...fileViewPaths, ...fullWorkspacePaths])].filter(include).sort(); + const visible: string[] = []; + for (const path of candidates) { + if (fileView.defaultReadState === "after") { + const overlay = fileView.overlayFor(path); + if (overlay?.action === "delete") continue; + if (overlay?.action === "write" + || (fileViewPathSet.has(path) && (fullWorkspace === undefined || fullWorkspacePathSet.has(path)))) { + visible.push(path); + continue; + } + } + if (await fileView.exists(path)) visible.push(path); + } + return visible; + }; return { read: async (path) => { const result = await fileView.readAfter(validateRepoRelativePath(path)); return result.status === "found" ? result.content : undefined; }, - list: async () => { - const fileViewPaths = await fileView.listVisibleFiles(); - const fullWorkspacePaths = fullWorkspace === undefined ? [] : await fullWorkspace.list(); - const fileViewPathSet = new Set(fileViewPaths); - const fullWorkspacePathSet = new Set(fullWorkspacePaths); - const candidates = [...new Set([...fileViewPaths, ...fullWorkspacePaths])] - .filter(isPythonProjectWorkspaceInput) - .sort(); - const visible: string[] = []; - for (const path of candidates) { - if (fileView.defaultReadState === "after") { - const overlay = fileView.overlayFor(path); - if (overlay?.action === "delete") continue; - if (overlay?.action === "write" || fileViewPathSet.has(path) || fullWorkspacePathSet.has(path)) { - visible.push(path); - continue; - } - } - if (await fileView.exists(path)) visible.push(path); - } - return visible; - }, + list: async () => listPaths(isPythonProjectWorkspaceInput), + listAll: async () => listPaths(() => true), exists: (path) => fileView.exists(validateRepoRelativePath(path)), realpath: async (path) => { if (path === ".") { @@ -67,7 +70,10 @@ export function createValidationFileViewPythonWorkspace( if (baseline.unavailable && await fullWorkspace.exists(normalized)) return baseline; return { path: normalized, symlink: false }; }, - executableExists: executableAvailable + executableExists: executableAvailable, + statMode: fullWorkspace?.statMode === undefined + ? undefined + : async (path) => fullWorkspace.statMode?.(validateRepoRelativePath(path)) }; } function isPythonProjectWorkspaceInput(path: string): boolean { @@ -115,7 +121,15 @@ export function createNodePythonProjectWorkspace(repoRoot: string): PythonProjec throw error; } }, - executableExists: nodeExecutableExists + executableExists: nodeExecutableExists, + statMode: async (path) => { + try { + return (await lstat(resolveRepoPath(canonicalRoot, path))).mode; + } catch (error) { + if (isMissing(error)) return undefined; + throw error; + } + } }; } diff --git a/packages/validation-python/src/pytest-check.ts b/packages/validation-python/src/pytest-check.ts new file mode 100644 index 0000000..9cce72f --- /dev/null +++ b/packages/validation-python/src/pytest-check.ts @@ -0,0 +1,157 @@ +import type { + GraphFactEdge, + PythonProjectContext, + PythonProjectToolProvenance, + PythonPytestValidationCapabilityRun, + ValidationDiagnostic +} from "@the-open-engine/opcore-contracts"; +import type { ValidationCheckContext, ValidationCheckDefinition } from "@the-open-engine/opcore-validation"; +import { PYTHON_PYTEST_CHECK_ID } from "./check-ids.js"; +import { pythonCheckAdapter, pythonCheckOwner, supportedPythonValidationScopes } from "./check-constants.js"; +import { createRelevantTestsGraphRequirements } from "./graph-requirements.js"; +import { executeProjectPytest } from "./pytest-project-runner.js"; +import { candidateFailure, disabledPytestRun, failureRun, notApplicableRun, unsupportedPytestTool, unsupportedRun } from "./pytest-result.js"; +import type { ProjectRunInput } from "./pytest-types.js"; +import { pytestWorkspaceCaps } from "./pytest-workspace.js"; +import { type PythonProjectWorkspace } from "./project-workspace.js"; +import { pythonInputSet, type PythonProjectContextResolver, type PythonSourceRootResolver } from "./source-files.js"; +import type { PythonValidationToolchainOptions } from "./toolchain.js"; + +interface CreatePytestCheckOptions extends Omit {} + +export { disabledPytestRun }; + +export function createPytestCheck( + options: CreatePytestCheckOptions = {}, + resolveContexts?: PythonProjectContextResolver, + resolveRoots?: PythonSourceRootResolver +): ValidationCheckDefinition { + const resolveRootPaths = resolveRoots ?? (async (context) => pythonInputSet(context)); + return { + id: PYTHON_PYTEST_CHECK_ID, + owner: pythonCheckOwner, + adapter: pythonCheckAdapter, + defaultSeverity: "warning", + supportedScopes: supportedPythonValidationScopes, + defaultScopes: [], + requiresGraph: true, + graphRequirements: createRelevantTestsGraphRequirements(resolveRootPaths), + inactiveResult: (_context, state) => + state === "disabled" + ? disabledPytestRun() + : { + status: "skipped", + diagnostics: [], + pythonCapabilityRuns: [{ + capability: "pytest", + checkId: PYTHON_PYTEST_CHECK_ID, + activation: "not_applicable", + outcome: "not_applicable", + message: "python.pytest was not requested; pytest execution is opt-in.", + selectionMode: "none" + }] + }, + run: async (context) => runPytestCheck(context, options.nodeWorkspace, resolveContexts) + }; +} + +async function runPytestCheck( + context: ValidationCheckContext, + nodeWorkspace: PythonProjectWorkspace | undefined, + resolveContexts: PythonProjectContextResolver | undefined +) { + const targets = pythonInputSet(context); + if (targets.length === 0) return notApplicableRun("No Python source targets selected."); + if (resolveContexts === undefined) { + return failureRun("PYTHON_PYTEST_CONTEXT_MISSING", "Canonical Python project context resolver is required.", "tool_failure"); + } + const rootContexts = await resolveContexts(context, targets); + const unresolved = rootContexts.find(isUnresolvedPytestContext); + if (unresolved !== undefined) { + return unsupportedRun(unresolved, `Python pytest execution requires a resolved project context for ${unresolved.target}.`); + } + const selectedCandidates = selectCandidatePaths(targets, await context.graph.testedBy()); + if (selectedCandidates.length === 0) { + return candidateFailure(rootContexts, [], "No TESTED_BY graph candidate tests matched the selected Python targets."); + } + if (selectedCandidates.length > pytestWorkspaceCaps.maxCandidateFiles) { + return failureRun("PYTHON_PYTEST_CANDIDATE_OVERFLOW", `Pytest candidate selection exceeded ${pytestWorkspaceCaps.maxCandidateFiles} files.`, "tool_failure", rootContexts, selectedCandidates); + } + const candidateContexts = await resolveContexts(context, selectedCandidates); + const groups = groupProjects(candidateContexts, selectedCandidates); + const diagnostics: ValidationDiagnostic[] = []; + const capabilityRuns: PythonPytestValidationCapabilityRun[] = []; + let overallPassed = true; + for (const group of groups) { + if (group.candidatePaths.some((path) => !candidateContexts.some((entry) => entry.target === path))) { + return failureRun("PYTHON_PYTEST_CANDIDATE_MISSING", "Pytest candidate context resolution omitted at least one candidate test.", "tool_failure", rootContexts, group.candidatePaths); + } + const pytest = selectPytestTool(group.context); + if (pytest === undefined) { + const result = unsupportedPytestTool(group.context, group.candidatePaths); + diagnostics.push(...(result.diagnostics ?? [])); + for (const capabilityRun of result.pythonCapabilityRuns ?? []) { + if (capabilityRun.capability === "pytest") capabilityRuns.push(capabilityRun); + } + overallPassed = false; + continue; + } + const run = await executeProjectPytest(context, nodeWorkspace, group, pytest); + diagnostics.push(...run.diagnostics); + capabilityRuns.push(run.capabilityRun); + if (run.outcome !== "passed") overallPassed = false; + } + const outcome: "passed" | "findings" = overallPassed && capabilityRuns.some((run) => run.counts?.passedCount && run.cleanup?.ok) + ? "passed" + : "findings"; + return { + diagnostics, + outcome, + pythonProjectContexts: rootContexts, + pythonCapabilityRuns: capabilityRuns + }; +} + +function selectCandidatePaths(rootPaths: readonly string[], testedBy: readonly GraphFactEdge[]): readonly string[] { + const selected = new Set(); + for (const edge of testedBy) { + const fromPath = endpointFilePath(edge.from); + const toPath = endpointFilePath(edge.to); + if (fromPath !== undefined && rootPaths.includes(fromPath) && toPath?.endsWith(".py")) selected.add(toPath); + if (toPath !== undefined && rootPaths.includes(toPath) && fromPath?.endsWith(".py")) selected.add(fromPath); + } + return [...selected].sort(); +} + +function groupProjects(contexts: readonly PythonProjectContext[], candidatePaths: readonly string[]): readonly ProjectRunInput[] { + const groups = new Map(); + for (const context of contexts) { + const group = groups.get(context.projectKey) ?? { context, candidatePaths: [] }; + if (candidatePaths.includes(context.target)) group.candidatePaths.push(context.target); + groups.set(context.projectKey, group); + } + return [...groups.values()] + .map((entry) => ({ context: entry.context, candidatePaths: [...new Set(entry.candidatePaths)].sort() })) + .filter((entry) => entry.candidatePaths.length > 0) + .sort((left, right) => left.context.projectRoot.localeCompare(right.context.projectRoot)); +} + +function selectPytestTool(context: PythonProjectContext): PythonProjectToolProvenance | undefined { + return context.tools.find((tool) => tool.tool === "pytest" && tool.available); +} + +function isUnresolvedPytestContext(context: PythonProjectContext): boolean { + if (context.interpreter === undefined || context.outcome === "ambiguous" || context.outcome === "unsupported") return true; + return context.reasons.some((reason) => + reason.code === "invalid_config" || + reason.code === "path_refused" || + reason.code === "symlink_refused" || + reason.code === "ambiguous_path" || + reason.tool === "python" + ); +} + +function endpointFilePath(endpoint: string): string | undefined { + const match = /^[^:]+:([^#]+)(?:#.*)?$/u.exec(endpoint); + return match?.[1]; +} diff --git a/packages/validation-python/src/pytest-hook-source.ts b/packages/validation-python/src/pytest-hook-source.ts new file mode 100644 index 0000000..10c5924 --- /dev/null +++ b/packages/validation-python/src/pytest-hook-source.ts @@ -0,0 +1,47 @@ +export function pytestHookSource(): string { + return [ + "import json", + "import os", + "from pathlib import Path", + "", + "_output_path = Path(os.environ['OPCORE_PYTEST_HOOK_OUTPUT'])", + "_selection_manifest = os.environ.get('OPCORE_PYTEST_SELECTION_MANIFEST')", + "_selected_ids = None", + "if _selection_manifest:", + " _selected_ids = set()", + " for line in Path(_selection_manifest).read_text(encoding='utf-8').splitlines():", + " line = line.strip()", + " if line:", + " _selected_ids.add(line)", + "", + "def _emit(payload):", + " with _output_path.open('a', encoding='utf-8') as handle:", + " handle.write(json.dumps(payload, sort_keys=True) + '\\n')", + "", + "def pytest_collection_modifyitems(session, config, items):", + " if _selected_ids is not None:", + " selected = [item for item in items if item.nodeid in _selected_ids]", + " items[:] = selected", + " for item in items:", + " _emit({'type': 'collected', 'nodeid': item.nodeid})", + "", + "def pytest_collectreport(report):", + " if report.failed:", + " _emit({'type': 'collect_error', 'nodeid': getattr(report, 'nodeid', ''), 'message': str(report.longrepr)})", + "", + "def pytest_runtest_logreport(report):", + " payload = {", + " 'type': 'test_report',", + " 'nodeid': report.nodeid,", + " 'when': report.when,", + " 'outcome': report.outcome", + " }", + " wasxfail = getattr(report, 'wasxfail', None)", + " if wasxfail is not None:", + " payload['wasxfail'] = str(wasxfail)", + " _emit(payload)", + "", + "def pytest_sessionfinish(session, exitstatus):", + " _emit({'type': 'session_finish', 'exitstatus': int(exitstatus)})" + ].join("\n"); +} diff --git a/packages/validation-python/src/pytest-process.ts b/packages/validation-python/src/pytest-process.ts new file mode 100644 index 0000000..1a735e7 --- /dev/null +++ b/packages/validation-python/src/pytest-process.ts @@ -0,0 +1,232 @@ +import { createHash } from "node:crypto"; +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { PythonCapabilityInvocation, PythonProjectToolProvenance } from "@the-open-engine/opcore-contracts"; +import { runTool, type PythonToolRunResult } from "./process.js"; +import { pytestHookSource } from "./pytest-hook-source.js"; +import { pytestDiagnostic } from "./pytest-result.js"; +import { + analyzeExecutionEvents, + collectErrorCount, + countResults, + executionFailureDiagnostics, + readHookReport, + uniqueCollectedNodeIds +} from "./pytest-protocol.js"; +import type { CollectionRunResult, ExecutionRunResult, ProjectRunInput } from "./pytest-types.js"; +import { pytestWorkspaceCaps, type MaterializedPytestWorkspace } from "./pytest-workspace.js"; + +export function writePytestRuntimeModule(runtimeRoot: string): string { + const runtimeModule = join(runtimeRoot, "opcore_pytest_hook.py"); + writeFileSync(runtimeModule, pytestHookSource(), "utf8"); + return runtimeModule; +} + +export async function runCollection( + materialized: MaterializedPytestWorkspace, + group: ProjectRunInput, + pytest: PythonProjectToolProvenance +): Promise { + const outputPath = join(materialized.runtimeRoot, "collection.jsonl"); + const selectionDigest = sha256(group.candidatePaths); + const args = [ + ...pytest.argv.slice(1), + "-p", "no:cacheprovider", + "-p", "opcore_pytest_hook", + "--collect-only", + "-q", + ...group.candidatePaths.map((path) => relativeProjectPath(path, group.context.projectRoot)) + ]; + const startedAt = Date.now(); + const result = await runTool(pytest.executable, args, { + cwd: materialized.projectCwd, + env: pytestEnv(materialized.runtimeRoot, outputPath), + timeoutMs: 30000, + allowedExitCodes: [0, 1, 2, 3, 4, 5], + maxOutputBytes: pytestWorkspaceCaps.maxProcessOutputBytes + }); + const invocation = invocationSummary("collection", pytest, args, result, "direct_argv", selectionDigest, Date.now() - startedAt); + const report = readHookReport(outputPath, { requireSessionFinish: result.termination === "exited" }); + return collectionOutcome(result, report, invocation, selectionDigest, group.candidatePaths.length); +} + +export async function runExecution( + materialized: MaterializedPytestWorkspace, + group: ProjectRunInput, + pytest: PythonProjectToolProvenance, + nodeIds: readonly string[] +): Promise { + const outputPath = join(materialized.runtimeRoot, "execution.jsonl"); + const selectionDigest = sha256(nodeIds); + const args = [...pytest.argv.slice(1), "-p", "no:cacheprovider", "-p", "opcore_pytest_hook", "-q"]; + const env = pytestEnv(materialized.runtimeRoot, outputPath); + const selectionMode = maybeAttachSelectionManifest(args, env, materialized.runtimeRoot, nodeIds); + if (selectionMode instanceof Error) { + return { + outcome: "tool_failure", + message: selectionMode.message, + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_SELECTION_OVERFLOW", selectionMode.message, group.context.target)], + counts: countResults(group.candidatePaths.length, nodeIds.length, 0, 0, 0, 0, 0, 0, 0), + invocation: invocationSummary("execution", pytest, args, spawnErrorResult(pytest.executable, args, materialized.projectCwd, selectionMode.message), "manifest", selectionDigest, 0), + selectionMode: "manifest", + selectionDigest + }; + } + const startedAt = Date.now(); + const result = await runTool(pytest.executable, args, { + cwd: materialized.projectCwd, + env, + timeoutMs: 30000, + allowedExitCodes: [0, 1, 2, 3, 4, 5], + maxOutputBytes: pytestWorkspaceCaps.maxProcessOutputBytes + }); + const invocation = invocationSummary("execution", pytest, args, result, selectionMode, selectionDigest, Date.now() - startedAt); + const report = readHookReport(outputPath, { requireSessionFinish: result.termination === "exited" }); + return executionOutcome(result, report, invocation, selectionMode, selectionDigest, group.context.target, group.candidatePaths.length, nodeIds); +} + +function collectionOutcome( + result: PythonToolRunResult, + report: ReturnType, + invocation: PythonCapabilityInvocation, + selectionDigest: string, + candidateCount: number +): CollectionRunResult { + const nodeIds = report.events.some((event) => event.type === "test_report") ? [] : uniqueCollectedNodeIds(report.events); + const counts = countResults(candidateCount, nodeIds.length, 0, 0, 0, 0, 0, 0, collectErrorCount(report.events)); + if (result.termination === "exited" && report.exitStatus !== result.exitCode) return failedCollection("tool_failure", `Pytest collection hook exit status ${report.exitStatus} did not match process exit code ${result.exitCode}.`, counts, invocation, selectionDigest); + if (report.events.some((event) => event.type === "test_report")) return failedCollection("tool_failure", "Pytest collection emitted test execution reports during --collect-only.", counts, invocation, selectionDigest); + if (result.termination === "timeout") return failedCollection("timeout", result.failureMessage ?? "pytest collection timed out", counts, invocation, selectionDigest); + if (result.termination !== "exited") return failedCollection("tool_failure", result.failureMessage ?? "pytest collection failed", counts, invocation, selectionDigest); + if (result.exitCode === 4) return failedCollection("invalid_config", "Pytest collection reported invalid configuration.", counts, invocation, selectionDigest); + if (result.exitCode !== 0 && result.exitCode !== 1) return failedCollection("tool_failure", `Pytest collection exited with code ${result.exitCode}.`, counts, invocation, selectionDigest); + if (result.exitCode === 1 && collectErrorCount(report.events) === 0) return failedCollection("tool_failure", "Pytest collection exited with code 1 but emitted no collection-error events.", counts, invocation, selectionDigest, nodeIds); + if (result.exitCode === 1) return failedCollection("findings", "Pytest collection reported collection errors.", counts, invocation, selectionDigest, nodeIds); + if (collectErrorCount(report.events) > 0) return failedCollection("tool_failure", "Pytest collection emitted collection-error events despite a zero exit code.", counts, invocation, selectionDigest); + if (nodeIds.length === 0) return failedCollection("findings", "Pytest collection produced zero node ids.", counts, invocation, selectionDigest); + if (nodeIds.length > pytestWorkspaceCaps.maxCollectedNodeIds) return failedCollection("tool_failure", `Pytest collection exceeded ${pytestWorkspaceCaps.maxCollectedNodeIds} node ids.`, counts, invocation, selectionDigest); + if (Buffer.byteLength(nodeIds.join("\n"), "utf8") > pytestWorkspaceCaps.maxNodeIdBytes) return failedCollection("tool_failure", `Pytest collection node ids exceeded ${pytestWorkspaceCaps.maxNodeIdBytes} bytes.`, counts, invocation, selectionDigest); + return { outcome: "passed", message: "Pytest collection succeeded.", nodeIds, counts, invocation, selectionMode: "direct_argv", selectionDigest }; +} + +function executionOutcome( + result: PythonToolRunResult, + report: ReturnType, + invocation: PythonCapabilityInvocation, + selectionMode: "direct_argv" | "manifest", + selectionDigest: string, + path: string, + candidateCount: number, + nodeIds: readonly string[] +): ExecutionRunResult { + if (result.termination === "exited" && report.exitStatus !== result.exitCode) return failedExecution("tool_failure", `Pytest execution hook exit status ${report.exitStatus} did not match process exit code ${result.exitCode}.`, [pytestDiagnostic("PYTHON_PYTEST_PROTOCOL_MISMATCH", `Pytest execution hook exit status ${report.exitStatus} did not match process exit code ${result.exitCode}.`, path)], countResults(candidateCount, nodeIds.length, 0, 0, 0, 0, 0, 0, 0), invocation, selectionMode, selectionDigest); + const analysis = analyzeExecutionEvents(candidateCount, nodeIds, report.events, path); + if (result.termination === "timeout") return failedExecution("timeout", result.failureMessage ?? "pytest execution timed out", [pytestDiagnostic("PYTHON_PYTEST_TIMEOUT", result.failureMessage ?? "pytest execution timed out", path)], analysis.counts, invocation, selectionMode, selectionDigest); + if (result.termination !== "exited") return failedExecution("tool_failure", result.failureMessage ?? "pytest execution failed", [pytestDiagnostic("PYTHON_PYTEST_TOOL_FAILED", result.failureMessage ?? "pytest execution failed", path)], analysis.counts, invocation, selectionMode, selectionDigest); + if (analysis.diagnostics.length > 0) return failedExecution("tool_failure", analysis.diagnostics[0]?.message ?? "Pytest execution hook protocol mismatch.", analysis.diagnostics, analysis.counts, invocation, selectionMode, selectionDigest); + if (analysis.counts.executedCount === 0) return failedExecution("findings", "Pytest execution ran zero tests.", [pytestDiagnostic("PYTHON_PYTEST_NO_EXECUTION", "Pytest execution ran zero tests.", path)], analysis.counts, invocation, selectionMode, selectionDigest); + if (analysis.counts.passedCount === 0) return failedExecution("findings", "Pytest execution produced no passing tests.", [pytestDiagnostic("PYTHON_PYTEST_NO_PASS", "Pytest execution produced no passing tests.", path)], analysis.counts, invocation, selectionMode, selectionDigest); + const errors = executionFailureDiagnostics(report.events, path); + if (result.exitCode !== 0 || errors.length > 0 || analysis.counts.failedCount > 0 || analysis.counts.xpassedCount > 0 || analysis.counts.errorCount > 0) { + return failedExecution("findings", "Pytest execution reported failing outcomes.", errors.length > 0 ? errors : [pytestDiagnostic("PYTHON_PYTEST_FAILURES", "Pytest execution reported failing outcomes.", path)], analysis.counts, invocation, selectionMode, selectionDigest); + } + return { outcome: "passed", message: "Pytest execution reported passing tests.", diagnostics: [], counts: analysis.counts, invocation, selectionMode, selectionDigest }; +} + +function failedCollection( + outcome: CollectionRunResult["outcome"], + message: string, + counts: CollectionRunResult["counts"], + invocation: CollectionRunResult["invocation"], + selectionDigest: string, + nodeIds: readonly string[] = [] +): CollectionRunResult { + return { outcome, message, nodeIds, counts, invocation, selectionMode: "direct_argv", selectionDigest }; +} + +function failedExecution( + outcome: ExecutionRunResult["outcome"], + message: string, + diagnostics: ExecutionRunResult["diagnostics"], + counts: ExecutionRunResult["counts"], + invocation: ExecutionRunResult["invocation"], + selectionMode: ExecutionRunResult["selectionMode"], + selectionDigest: string +): ExecutionRunResult { + return { outcome, message, diagnostics, counts, invocation, selectionMode, selectionDigest }; +} + +function maybeAttachSelectionManifest( + args: string[], + env: Record, + runtimeRoot: string, + nodeIds: readonly string[] +): "direct_argv" | "manifest" | Error { + if (Buffer.byteLength(nodeIds.join("\0"), "utf8") <= pytestWorkspaceCaps.maxArgvBytes) { + args.push(...nodeIds); + return "direct_argv"; + } + const manifest = `${nodeIds.join("\n")}\n`; + if (Buffer.byteLength(manifest, "utf8") > pytestWorkspaceCaps.maxManifestBytes) { + return new Error(`Pytest selection manifest exceeded ${pytestWorkspaceCaps.maxManifestBytes} bytes.`); + } + const manifestPath = join(runtimeRoot, "selection.txt"); + writeFileSync(manifestPath, manifest, "utf8"); + env.OPCORE_PYTEST_SELECTION_MANIFEST = manifestPath; + return "manifest"; +} + +function pytestEnv(runtimeRoot: string, outputPath: string): Record { + return { ...process.env, PYTHONPATH: runtimeRoot, PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1", OPCORE_PYTEST_HOOK_OUTPUT: outputPath }; +} + +function invocationSummary( + stage: "collection" | "execution", + pytest: PythonProjectToolProvenance, + args: readonly string[], + result: PythonToolRunResult, + selectionMode: "direct_argv" | "manifest", + selectionDigest: string, + durationMs: number +): PythonCapabilityInvocation { + return { + stage, + command: pytest.tool, + argsDigest: sha256([pytest.executable, ...args]), + argCount: args.length, + selectionMode, + selectionDigest, + durationMs, + termination: result.termination, + ...(result.exitCode === null ? {} : { exitCode: result.exitCode }), + ...(result.signal === null ? {} : { signal: result.signal }), + outputBytes: Buffer.byteLength(result.stdout, "utf8") + Buffer.byteLength(result.stderr, "utf8"), + stdoutDigest: sha256(result.stdout), + stderrDigest: sha256(result.stderr) + }; +} + +function relativeProjectPath(path: string, projectRoot: string): string { + return projectRoot === "." ? path : path.slice(`${projectRoot}/`.length); +} + +function sha256(value: unknown): string { + const input = typeof value === "string" ? value : JSON.stringify(value); + return `sha256:${createHash("sha256").update(input, "utf8").digest("hex")}`; +} + +function spawnErrorResult(command: string, args: readonly string[], cwd: string, failureMessage: string): PythonToolRunResult { + return { + ok: false, + termination: "spawn_error", + command, + args: [...args], + cwd, + allowedExitCodes: [0], + exitCode: null, + signal: null, + stdout: "", + stderr: "", + failureMessage + }; +} diff --git a/packages/validation-python/src/pytest-project-runner.ts b/packages/validation-python/src/pytest-project-runner.ts new file mode 100644 index 0000000..eb32b5b --- /dev/null +++ b/packages/validation-python/src/pytest-project-runner.ts @@ -0,0 +1,123 @@ +import type { PythonProjectToolProvenance, PythonPytestValidationCapabilityRun } from "@the-open-engine/opcore-contracts"; +import type { ValidationCheckContext } from "@the-open-engine/opcore-validation"; +import type { PythonProjectWorkspace } from "./project-workspace.js"; +import { collectPytestProjectPaths, materializePytestWorkspace } from "./pytest-workspace.js"; +import { runCollection, runExecution, writePytestRuntimeModule } from "./pytest-process.js"; +import { cleanupFailureMessage, createCleanupState, finalizeCapabilityRun, pytestDiagnostic, recordCleanup } from "./pytest-result.js"; +import type { ProjectRunInput, ProjectRunResult } from "./pytest-types.js"; +import { PYTHON_PYTEST_CHECK_ID } from "./check-ids.js"; + +export async function executeProjectPytest( + validation: ValidationCheckContext, + nodeWorkspace: PythonProjectWorkspace | undefined, + group: ProjectRunInput, + pytest: PythonProjectToolProvenance +): Promise { + const projectPaths = await collectPytestProjectPaths(validation, group.context, group.candidatePaths, nodeWorkspace); + const capabilityBase = { + capability: "pytest" as const, + checkId: PYTHON_PYTEST_CHECK_ID, + activation: "enabled" as const, + projectKey: group.context.projectKey, + projectRoot: group.context.projectRoot, + configFile: pytest.configFile, + targetCount: group.candidatePaths.length, + candidatePaths: group.candidatePaths + }; + const cleanup = createCleanupState(); + let afterStateFingerprint: string | undefined; + let collectedNodeIds: readonly string[] | undefined; + let collectionSelectionMode: "direct_argv" | "manifest" | "none" = "none"; + let collectionSelectionDigest: string | undefined; + let collectionCounts; + let collectionInvocation; + try { + const collectionWorkspace = await materializePytestWorkspace({ context: validation, project: group.context, paths: projectPaths, nodeWorkspace }); + afterStateFingerprint = collectionWorkspace.afterStateFingerprint; + let collectionFailure: ProjectRunResult | undefined; + try { + writePytestRuntimeModule(collectionWorkspace.runtimeRoot); + const collection = await runCollection(collectionWorkspace, group, pytest); + collectionInvocation = collection.invocation; + collectionCounts = collection.counts; + collectionSelectionMode = collection.selectionMode; + collectionSelectionDigest = collection.selectionDigest; + collectedNodeIds = collection.nodeIds; + if (collection.outcome !== "passed") { + collectionFailure = { + outcome: collection.outcome, + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_COLLECTION_FAILED", collection.message, group.context.target)], + capabilityRun: { ...capabilityBase, afterStateFingerprint, outcome: collection.outcome, message: collection.message, selectionMode: collection.selectionMode, selectionDigest: collection.selectionDigest, counts: collection.counts, collection: collection.invocation } + }; + } + } finally { + recordCleanup(cleanup, collectionWorkspace.cleanup); + } + if (collectionFailure !== undefined) return finalizeCapabilityRun(cleanup, collectionFailure); + if (!cleanup.ok) return cleanupFailureResult(cleanup, capabilityBase, afterStateFingerprint, collectedNodeIds, collectionSelectionMode, collectionSelectionDigest, collectionCounts, collectionInvocation, group.context.target); + const executionWorkspace = await materializePytestWorkspace({ context: validation, project: group.context, paths: projectPaths, nodeWorkspace }); + if (executionWorkspace.afterStateFingerprint !== afterStateFingerprint) { + recordCleanup(cleanup, executionWorkspace.cleanup); + return finalizeCapabilityRun(cleanup, { + outcome: "tool_failure", + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_FINGERPRINT_MISMATCH", "Pytest execution after-state fingerprint did not match collection.", group.context.target)], + capabilityRun: { ...capabilityBase, afterStateFingerprint, outcome: "tool_failure", message: "Pytest execution after-state fingerprint did not match collection.", collectedNodeIds, selectionMode: collectionSelectionMode, selectionDigest: collectionSelectionDigest, counts: collectionCounts, collection: collectionInvocation } + }); + } + let executionResult: ProjectRunResult | undefined; + try { + writePytestRuntimeModule(executionWorkspace.runtimeRoot); + const execution = await runExecution(executionWorkspace, group, pytest, collectedNodeIds ?? []); + executionResult = { + outcome: execution.outcome, + diagnostics: execution.diagnostics, + capabilityRun: { ...capabilityBase, afterStateFingerprint, outcome: execution.outcome, message: execution.message, collectedNodeIds, selectionMode: execution.selectionMode, selectionDigest: execution.selectionDigest, counts: execution.counts, collection: collectionInvocation, execution: execution.invocation } + }; + } finally { + recordCleanup(cleanup, executionWorkspace.cleanup); + } + return finalizeCapabilityRun(cleanup, executionResult ?? { + outcome: "tool_failure", + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_TOOL_FAILED", "Pytest execution produced no result.", group.context.target)], + capabilityRun: { ...capabilityBase, afterStateFingerprint, outcome: "tool_failure", message: "Pytest execution produced no result.", collectedNodeIds, selectionMode: collectionSelectionMode, selectionDigest: collectionSelectionDigest, counts: collectionCounts, collection: collectionInvocation } + }); + } catch (error) { + return finalizeCapabilityRun(cleanup, { + outcome: "tool_failure", + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_TOOL_FAILED", errorMessage(error), group.context.target)], + capabilityRun: { ...capabilityBase, afterStateFingerprint, outcome: "tool_failure", message: errorMessage(error), collectedNodeIds, selectionMode: collectionSelectionMode, selectionDigest: collectionSelectionDigest, counts: collectionCounts, collection: collectionInvocation } + }); + } +} + +function cleanupFailureResult( + cleanup: Parameters[0], + capabilityBase: { + capability: "pytest"; + checkId: string; + activation: "enabled"; + projectKey: string; + projectRoot: string; + configFile?: string; + targetCount: number; + candidatePaths: readonly string[]; + }, + afterStateFingerprint: string | undefined, + collectedNodeIds: readonly string[] | undefined, + selectionMode: "direct_argv" | "manifest" | "none", + selectionDigest: string | undefined, + counts: PythonPytestValidationCapabilityRun["counts"], + collection: PythonPytestValidationCapabilityRun["collection"], + path: string +): ProjectRunResult { + const message = cleanupFailureMessage(cleanup); + return finalizeCapabilityRun(cleanup, { + outcome: "tool_failure", + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_CLEANUP_FAILED", message, path)], + capabilityRun: { ...capabilityBase, afterStateFingerprint, outcome: "tool_failure", message, collectedNodeIds, selectionMode, selectionDigest, counts, collection } + }); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/validation-python/src/pytest-protocol.ts b/packages/validation-python/src/pytest-protocol.ts new file mode 100644 index 0000000..903f83b --- /dev/null +++ b/packages/validation-python/src/pytest-protocol.ts @@ -0,0 +1,224 @@ +import type { PythonCapabilityCounts, ValidationDiagnostic } from "@the-open-engine/opcore-contracts"; +import { readFileSync } from "node:fs"; +import { pytestWorkspaceCaps } from "./pytest-workspace.js"; +import { pytestDiagnostic } from "./pytest-result.js"; +import type { ExecutionEventAnalysis, HookEvent, HookReport } from "./pytest-types.js"; + +export function readHookReport(path: string, options: { requireSessionFinish?: boolean } = {}): HookReport { + const requireSessionFinish = options.requireSessionFinish ?? true; + try { + const text = readFileSync(path, "utf8"); + if (Buffer.byteLength(text, "utf8") > pytestWorkspaceCaps.maxHookReportBytes) { + throw new Error(`Pytest hook report exceeded ${pytestWorkspaceCaps.maxHookReportBytes} bytes.`); + } + const events: HookEvent[] = []; + let exitStatus: number | undefined; + for (const [index, line] of text.split(/\r?\n/u).filter(Boolean).entries()) { + const payload = JSON.parse(line) as unknown; + if (!isRecord(payload)) throw new Error(`Pytest hook event ${index + 1} is not an object.`); + if (payload.type === "session_finish") { + if (exitStatus !== undefined) throw new Error("Duplicate pytest session_finish event."); + if (!Number.isInteger(payload.exitstatus)) throw new Error("Pytest session_finish event requires integer exitstatus."); + exitStatus = payload.exitstatus as number; + continue; + } + events.push(validateHookEvent(payload, index + 1)); + } + if (exitStatus === undefined) { + if (!requireSessionFinish) return { events, exitStatus: -1 }; + throw new Error("Pytest hook report omitted session_finish."); + } + return { events, exitStatus }; + } catch (error) { + if ((error as { code?: string }).code === "ENOENT") { + if (!requireSessionFinish) return { events: [], exitStatus: -1 }; + throw new Error("Pytest hook report was not created."); + } + throw error; + } +} + +export function uniqueCollectedNodeIds(events: readonly HookEvent[]): readonly string[] { + const nodeIds = new Set(); + for (const event of events) { + if (event.type !== "collected" || event.nodeid === undefined) continue; + if (nodeIds.has(event.nodeid)) throw new Error(`Duplicate pytest collection node id: ${event.nodeid}`); + nodeIds.add(event.nodeid); + } + return [...nodeIds]; +} + +export function collectErrorCount(events: readonly HookEvent[]): number { + return events.filter((event) => event.type === "collect_error").length; +} + +export function analyzeExecutionEvents( + candidateCount: number, + expectedNodeIds: readonly string[], + events: readonly HookEvent[], + path: string +): ExecutionEventAnalysis { + const expected = new Set(expectedNodeIds); + const collected = new Set(); + const terminal = new Map(); + const diagnostics: ValidationDiagnostic[] = []; + for (const event of events) processExecutionEvent(event, expected, collected, terminal, diagnostics, path); + if (collected.size !== expected.size || expectedNodeIds.some((nodeId) => !collected.has(nodeId))) { + diagnostics.push(pytestDiagnostic("PYTHON_PYTEST_PROTOCOL_MISMATCH", "Pytest execution collected node ids did not match the collection selection.", path)); + } + const counts = summarizeExecutionCounts(candidateCount, expectedNodeIds.length, terminal, events); + if (counts.executedCount !== expectedNodeIds.length && collectErrorCount(events) === 0) { + diagnostics.push(pytestDiagnostic("PYTHON_PYTEST_PROTOCOL_MISMATCH", "Pytest execution terminal node ids did not match the collected selection.", path)); + } + return { counts, diagnostics }; +} + +export function executionFailureDiagnostics(events: readonly HookEvent[], path: string) { + return events + .filter((event) => event.type === "collect_error" && event.message !== undefined) + .map((event) => pytestDiagnostic("PYTHON_PYTEST_COLLECTION_ERROR", event.message!, path)); +} + +export function countResults( + candidateCount: number, + collectedCount: number, + executedCount: number, + passedCount: number, + failedCount: number, + skippedCount: number, + xfailedCount: number, + xpassedCount: number, + errorCount: number +): PythonCapabilityCounts { + return { candidateCount, collectedCount, executedCount, passedCount, failedCount, skippedCount, xfailedCount, xpassedCount, errorCount }; +} + +interface NodeOutcomeState { + callSeen: boolean; + passed: boolean; + failed: boolean; + skipped: boolean; + xfailed: boolean; + xpassed: boolean; + error: boolean; + setupSkipped: boolean; +} + +function processExecutionEvent( + event: HookEvent, + expected: ReadonlySet, + collected: Set, + terminal: Map, + diagnostics: ReturnType, + path: string +): void { + if (event.type === "collected") return processCollectedEvent(event, expected, collected, diagnostics, path); + if (event.type !== "test_report" || event.nodeid === undefined || event.when === undefined || event.outcome === undefined) return; + if (!expected.has(event.nodeid)) { + diagnostics.push(pytestDiagnostic("PYTHON_PYTEST_PROTOCOL_MISMATCH", `Pytest execution reported unexpected node id: ${event.nodeid}`, path)); + return; + } + const current = terminal.get(event.nodeid) ?? emptyNodeOutcomeState(); + if (event.when === "call") { + processCallOutcome( + event as HookEvent & { nodeid: string; when: "call"; outcome: "passed" | "failed" | "skipped" }, + current, + diagnostics, + path, + terminal + ); + return; + } + if (event.outcome === "failed") current.error = true; + if (event.when === "setup" && event.outcome === "skipped") current.setupSkipped = true; + terminal.set(event.nodeid, current); +} + +function processCollectedEvent( + event: HookEvent, + expected: ReadonlySet, + collected: Set, + diagnostics: ReturnType, + path: string +): void { + if (event.nodeid === undefined || !expected.has(event.nodeid)) { + diagnostics.push(pytestDiagnostic("PYTHON_PYTEST_PROTOCOL_MISMATCH", `Pytest execution collected unexpected node id: ${event.nodeid ?? ""}`, path)); + return; + } + if (collected.has(event.nodeid)) { + diagnostics.push(pytestDiagnostic("PYTHON_PYTEST_PROTOCOL_MISMATCH", `Pytest execution collected duplicate node id: ${event.nodeid}`, path)); + return; + } + collected.add(event.nodeid); +} + +function processCallOutcome( + event: HookEvent & { nodeid: string; when: "call"; outcome: "passed" | "failed" | "skipped" }, + current: NodeOutcomeState, + diagnostics: ReturnType, + path: string, + terminal: Map +): void { + if (current.callSeen) { + diagnostics.push(pytestDiagnostic("PYTHON_PYTEST_PROTOCOL_MISMATCH", `Pytest execution reported duplicate call outcome for ${event.nodeid}.`, path)); + return; + } + current.callSeen = true; + if (event.outcome === "passed" && event.wasxfail === undefined) current.passed = true; + else if (event.outcome === "failed" && event.wasxfail !== undefined) current.xpassed = true; + else if (event.outcome === "failed") current.failed = true; + else if (event.outcome === "skipped" && event.wasxfail !== undefined) current.xfailed = true; + else if (event.outcome === "skipped") current.skipped = true; + terminal.set(event.nodeid, current); +} + +function summarizeExecutionCounts( + candidateCount: number, + collectedCount: number, + terminal: ReadonlyMap, + events: readonly HookEvent[] +): PythonCapabilityCounts { + let passedCount = 0; + let failedCount = 0; + let skippedCount = 0; + let xfailedCount = 0; + let xpassedCount = 0; + let errorCount = collectErrorCount(events); + for (const counts of terminal.values()) { + if (counts.passed) passedCount += 1; + if (counts.failed) failedCount += 1; + if (counts.skipped || counts.setupSkipped) skippedCount += 1; + if (counts.xfailed) xfailedCount += 1; + if (counts.xpassed) xpassedCount += 1; + if (counts.error) errorCount += 1; + } + return countResults(candidateCount, collectedCount, terminal.size, passedCount, failedCount, skippedCount, xfailedCount, xpassedCount, errorCount); +} + +function emptyNodeOutcomeState(): NodeOutcomeState { + return { callSeen: false, passed: false, failed: false, skipped: false, xfailed: false, xpassed: false, error: false, setupSkipped: false }; +} + +function validateHookEvent(payload: Record, lineNumber: number): HookEvent { + if (payload.type === "collected") { + if (typeof payload.nodeid !== "string" || payload.nodeid.length === 0) throw new Error(`Pytest collected event ${lineNumber} requires non-empty nodeid.`); + return { type: "collected", nodeid: payload.nodeid }; + } + if (payload.type === "collect_error") { + if (typeof payload.message !== "string" || payload.message.length === 0) throw new Error(`Pytest collect_error event ${lineNumber} requires message.`); + if (payload.nodeid !== undefined && typeof payload.nodeid !== "string") throw new Error(`Pytest collect_error event ${lineNumber} has invalid nodeid.`); + return { type: "collect_error", message: payload.message, ...(typeof payload.nodeid === "string" ? { nodeid: payload.nodeid } : {}) }; + } + if (payload.type === "test_report") { + if (typeof payload.nodeid !== "string" || payload.nodeid.length === 0) throw new Error(`Pytest test_report event ${lineNumber} requires non-empty nodeid.`); + if (payload.when !== "setup" && payload.when !== "call" && payload.when !== "teardown") throw new Error(`Pytest test_report event ${lineNumber} has invalid when value.`); + if (payload.outcome !== "passed" && payload.outcome !== "failed" && payload.outcome !== "skipped") throw new Error(`Pytest test_report event ${lineNumber} has invalid outcome.`); + if (payload.wasxfail !== undefined && typeof payload.wasxfail !== "string") throw new Error(`Pytest test_report event ${lineNumber} has invalid wasxfail value.`); + return { type: "test_report", nodeid: payload.nodeid, when: payload.when, outcome: payload.outcome, ...(typeof payload.wasxfail === "string" ? { wasxfail: payload.wasxfail } : {}) }; + } + throw new Error(`Unknown pytest hook event type at line ${lineNumber}: ${String(payload.type)}`); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} diff --git a/packages/validation-python/src/pytest-result.ts b/packages/validation-python/src/pytest-result.ts new file mode 100644 index 0000000..d410612 --- /dev/null +++ b/packages/validation-python/src/pytest-result.ts @@ -0,0 +1,170 @@ +import type { PythonProjectContext, PythonPytestValidationCapabilityRun, ValidationDiagnostic } from "@the-open-engine/opcore-contracts"; +import type { ValidationCheckResult } from "@the-open-engine/opcore-validation"; +import { PYTHON_PYTEST_CHECK_ID } from "./check-ids.js"; +import { diagnostic } from "./diagnostics.js"; +import type { CleanupState, ProjectRunResult, PytestRunOutcome } from "./pytest-types.js"; + +export function pytestDiagnostic(code: string, message: string, path: string | undefined): ValidationDiagnostic { + return diagnostic({ + category: "test", + severity: "warning", + code, + message, + ...(path === undefined ? {} : { path }) + }); +} + +export function notApplicableRun(message: string): ValidationCheckResult { + return { + diagnostics: [], + outcome: "passed", + pythonCapabilityRuns: [{ + capability: "pytest", + checkId: PYTHON_PYTEST_CHECK_ID, + activation: "not_applicable", + outcome: "not_applicable", + message + }] + }; +} + +export function disabledPytestRun(message = "python.pytest is disabled by repo policy."): ValidationCheckResult { + return { + diagnostics: [], + outcome: "passed", + pythonCapabilityRuns: [{ + capability: "pytest", + checkId: PYTHON_PYTEST_CHECK_ID, + activation: "disabled", + outcome: "disabled", + message, + selectionMode: "none" + }] + }; +} + +export function unsupportedRun(context: PythonProjectContext, message: string): ValidationCheckResult { + return { + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_CONTEXT_UNSUPPORTED", message, context.target)], + outcome: "unsupported_target", + pythonProjectContexts: [context], + pythonCapabilityRuns: [{ + capability: "pytest", + checkId: PYTHON_PYTEST_CHECK_ID, + activation: "enabled", + outcome: "unsupported_target", + message, + projectKey: context.projectKey, + projectRoot: context.projectRoot + }] + }; +} + +export function unsupportedPytestTool(context: PythonProjectContext, candidatePaths: readonly string[]): ValidationCheckResult { + return { + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_UNSUPPORTED", `pytest is unavailable for project ${context.projectRoot}.`, context.target)], + outcome: "tool_unavailable", + pythonProjectContexts: [context], + pythonCapabilityRuns: [{ + capability: "pytest", + checkId: PYTHON_PYTEST_CHECK_ID, + activation: "enabled", + outcome: "tool_unavailable", + message: `pytest is unavailable for project ${context.projectRoot}.`, + projectKey: context.projectKey, + projectRoot: context.projectRoot, + candidatePaths + }] + }; +} + +export function candidateFailure( + contexts: readonly PythonProjectContext[], + candidatePaths: readonly string[], + message: string +): ValidationCheckResult { + return { + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_NO_CANDIDATES", message, contexts[0]?.target)], + outcome: "findings", + pythonProjectContexts: contexts, + pythonCapabilityRuns: [{ + capability: "pytest", + checkId: PYTHON_PYTEST_CHECK_ID, + activation: "enabled", + outcome: "no_candidates", + message, + candidatePaths + }] + }; +} + +export function failureRun( + code: string, + message: string, + outcome: Extract, + contexts: readonly PythonProjectContext[] = [], + candidatePaths: readonly string[] = [] +): ValidationCheckResult { + return { + diagnostics: [pytestDiagnostic(code, message, contexts[0]?.target)], + outcome, + pythonProjectContexts: contexts, + pythonCapabilityRuns: [{ + capability: "pytest", + checkId: PYTHON_PYTEST_CHECK_ID, + activation: "enabled", + outcome, + message, + candidatePaths + }] + }; +} + +export function createCleanupState(): CleanupState { + return { attempted: false, ok: true, failureMessages: [] }; +} + +export function recordCleanup(cleanup: CleanupState, action: () => void): void { + cleanup.attempted = true; + try { + action(); + } catch (error) { + cleanup.ok = false; + cleanup.failureMessages.push(errorMessage(error)); + } +} + +export function cleanupFailureMessage(cleanup: CleanupState): string { + return cleanup.failureMessages.length === 0 + ? "Pytest temporary workspace cleanup failed." + : `Pytest temporary workspace cleanup failed: ${cleanup.failureMessages.join("; ")}`; +} + +export function finalizeCapabilityRun(cleanup: CleanupState, result: ProjectRunResult): ProjectRunResult { + const cleanupEvidence: NonNullable = { + attempted: cleanup.attempted, + ok: cleanup.ok, + ...(cleanup.ok || cleanup.failureMessages.length === 0 ? {} : { failureMessage: cleanup.failureMessages.join("; ") }) + }; + if (cleanup.ok) { + return { ...result, capabilityRun: { ...result.capabilityRun, cleanup: cleanupEvidence } }; + } + const cleanupMessage = cleanupFailureMessage(cleanup); + return { + outcome: "tool_failure", + diagnostics: [ + ...result.diagnostics, + pytestDiagnostic("PYTHON_PYTEST_CLEANUP_FAILED", cleanupMessage, result.diagnostics[0]?.path ?? result.capabilityRun.projectRoot) + ], + capabilityRun: { + ...result.capabilityRun, + outcome: "tool_failure", + message: result.outcome === "passed" ? cleanupMessage : `${result.capabilityRun.message} Cleanup also failed.`, + cleanup: cleanupEvidence + } + }; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/validation-python/src/pytest-types.ts b/packages/validation-python/src/pytest-types.ts new file mode 100644 index 0000000..0abca1d --- /dev/null +++ b/packages/validation-python/src/pytest-types.ts @@ -0,0 +1,65 @@ +import type { + PythonCapabilityCounts, + PythonCapabilityInvocation, + PythonProjectContext, + PythonPytestValidationCapabilityRun, + ValidationDiagnostic +} from "@the-open-engine/opcore-contracts"; + +export type PytestRunOutcome = "passed" | "findings" | "tool_failure" | "timeout" | "invalid_config"; + +export interface ProjectRunInput { + context: PythonProjectContext; + candidatePaths: readonly string[]; +} + +export interface CleanupState { + attempted: boolean; + ok: boolean; + failureMessages: string[]; +} + +export interface HookEvent { + type: "collected" | "collect_error" | "test_report"; + nodeid?: string; + when?: "setup" | "call" | "teardown"; + outcome?: "passed" | "failed" | "skipped"; + wasxfail?: string; + message?: string; +} + +export interface HookReport { + events: readonly HookEvent[]; + exitStatus: number; +} + +export interface ExecutionEventAnalysis { + counts: PythonCapabilityCounts; + diagnostics: ValidationDiagnostic[]; +} + +export interface ProjectRunResult { + outcome: PytestRunOutcome; + diagnostics: ValidationDiagnostic[]; + capabilityRun: PythonPytestValidationCapabilityRun; +} + +export interface CollectionRunResult { + outcome: PytestRunOutcome; + message: string; + nodeIds: readonly string[]; + counts: PythonCapabilityCounts; + invocation: PythonCapabilityInvocation; + selectionMode: "direct_argv"; + selectionDigest: string; +} + +export interface ExecutionRunResult { + outcome: Exclude; + message: string; + diagnostics: ValidationDiagnostic[]; + counts: PythonCapabilityCounts; + invocation: PythonCapabilityInvocation; + selectionMode: "direct_argv" | "manifest"; + selectionDigest: string; +} diff --git a/packages/validation-python/src/pytest-workspace.ts b/packages/validation-python/src/pytest-workspace.ts new file mode 100644 index 0000000..074d49b --- /dev/null +++ b/packages/validation-python/src/pytest-workspace.ts @@ -0,0 +1,116 @@ +import { mkdtempSync } from "node:fs"; +import { mkdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { ValidationCheckContext } from "@the-open-engine/opcore-validation"; +import type { PythonProjectContext } from "@the-open-engine/opcore-contracts"; +import { + removeMaterializedWorkspace, + resolveMaterializedWorkspacePath, + writeMaterializedWorkspaceFile +} from "./materialized-workspace.js"; +import { pythonProjectDigest } from "./project-fingerprint.js"; +import { createValidationFileViewPythonWorkspace, type PythonProjectWorkspace } from "./project-workspace.js"; +import { isRelevantPythonConfig } from "./project-config-files.js"; + +export const pytestWorkspaceCaps = { + maxCandidateFiles: 128, + maxCollectedNodeIds: 2048, + maxNodeIdBytes: 256 * 1024, + maxManifestBytes: 256 * 1024, + maxHookReportBytes: 512 * 1024, + maxProcessOutputBytes: 512 * 1024, + maxWorkspaceBytes: 4 * 1024 * 1024, + maxArgvBytes: 16 * 1024 +} as const; + +export interface MaterializedPytestWorkspace { + root: string; + repoRoot: string; + projectCwd: string; + runtimeRoot: string; + afterStateFingerprint: string; + totalBytes: number; + cleanup(): void; +} + +export interface MaterializePytestWorkspaceArgs { + context: ValidationCheckContext; + project: PythonProjectContext; + paths: readonly string[]; + nodeWorkspace?: PythonProjectWorkspace; +} + +export async function materializePytestWorkspace( + args: MaterializePytestWorkspaceArgs +): Promise { + const workspace = createValidationFileViewPythonWorkspace(args.context.fileView, undefined, args.nodeWorkspace); + const tempRoot = mkdtempSync(join(tmpdir(), "opcore-python-pytest-")); + const repoRoot = join(tempRoot, "repo"); + const runtimeRoot = join(tempRoot, "runtime"); + let totalBytes = 0; + try { + await mkdir(repoRoot, { recursive: true }); + await mkdir(runtimeRoot, { recursive: true }); + const normalizedPaths = [...new Set(args.paths)].sort(); + const fingerprintEntries: { path: string; content: string }[] = []; + for (const path of normalizedPaths) { + const real = await workspace.realpath(path); + if (real.unavailable) throw new Error(`Pytest workspace realpath evidence is unavailable: ${path}`); + if (real.symlink || real.path !== path) throw new Error(`Pytest workspace refuses symlinked path: ${path}`); + const content = await workspace.read(path); + if (content === undefined) throw new Error(`Pytest workspace materialization requires visible file: ${path}`); + totalBytes += Buffer.byteLength(content, "utf8"); + if (totalBytes > pytestWorkspaceCaps.maxWorkspaceBytes) { + throw new Error(`Pytest workspace exceeded ${pytestWorkspaceCaps.maxWorkspaceBytes} bytes`); + } + fingerprintEntries.push({ path, content }); + await writeMaterializedWorkspaceFile(repoRoot, path, content, "pytest workspace", await workspace.statMode?.(path) ?? 0o644); + } + const projectCwd = args.project.projectRoot === "." ? repoRoot : writeMaterializedProjectPath(repoRoot, args.project.projectRoot); + return { + root: tempRoot, + repoRoot, + projectCwd, + runtimeRoot, + totalBytes, + afterStateFingerprint: pythonProjectDigest(fingerprintEntries), + cleanup: () => removeMaterializedWorkspace(tempRoot, "Pytest temporary workspace") + }; + } catch (error) { + removeMaterializedWorkspace(tempRoot, "Pytest temporary workspace"); + throw error; + } +} + +export async function collectPytestProjectPaths( + context: ValidationCheckContext, + project: PythonProjectContext, + candidatePaths: readonly string[], + nodeWorkspace?: PythonProjectWorkspace +): Promise { + const workspace = createValidationFileViewPythonWorkspace(context.fileView, undefined, nodeWorkspace); + const visible = workspace.listAll === undefined ? await workspace.list() : await workspace.listAll(); + const relevant = new Set(); + for (const path of visible) { + if (!withinProject(path, project.projectRoot)) continue; + relevant.add(path); + } + for (const candidate of candidatePaths) { + if (withinProject(candidate, project.projectRoot)) relevant.add(candidate); + } + for (const evidence of project.evidence) { + if (withinProject(evidence.path, project.projectRoot) && (isRelevantPythonConfig(evidence.path) || evidence.role === "config")) { + relevant.add(evidence.path); + } + } + return [...relevant].sort(); +} + +function withinProject(path: string, projectRoot: string): boolean { + return projectRoot === "." || path === projectRoot || path.startsWith(`${projectRoot}/`); +} + +function writeMaterializedProjectPath(root: string, path: string): string { + return path === "." ? root : resolveMaterializedWorkspacePath(root, path, "pytest workspace"); +} diff --git a/packages/validation-python/src/relevant-tests-check.ts b/packages/validation-python/src/relevant-tests-check.ts index 2a648a5..ddb5ab0 100644 --- a/packages/validation-python/src/relevant-tests-check.ts +++ b/packages/validation-python/src/relevant-tests-check.ts @@ -23,16 +23,16 @@ export function createRelevantTestsCheck(resolveRoots: PythonSourceRootResolver) category: "test", severity: "info", path, - code: "PY_RELEVANT_TESTS_FOUND", - message: `TESTED_BY graph evidence exists for ${path}: ${evidence.map(testEndpoint).sort().join(", ")}` + code: "PY_RELEVANT_TEST_CANDIDATES_FOUND", + message: `TESTED_BY graph candidate evidence for ${path}: ${evidence.map(testEndpoint).sort().join(", ")}` }; } return { category: "test", severity: "info", path, - code: "PY_RELEVANT_TESTS_ABSENT", - message: `No TESTED_BY graph evidence found for ${path}.` + code: "PY_RELEVANT_TEST_CANDIDATES_ABSENT", + message: `No TESTED_BY graph candidate evidence found for ${path}.` }; }); return { diagnostics }; diff --git a/packages/validation/src/aggregation.ts b/packages/validation/src/aggregation.ts index 1d94ce6..433538b 100644 --- a/packages/validation/src/aggregation.ts +++ b/packages/validation/src/aggregation.ts @@ -92,7 +92,7 @@ function pythonCapabilitySortKey(run: PythonValidationCapabilityRun): string { checkId: run.checkId, capability: run.capability, projectKey: run.projectKey ?? "", - contextFingerprint: run.contextFingerprint ?? "", + contextFingerprint: run.capability === "pytest" ? "" : run.contextFingerprint ?? "", receipt: run }); } diff --git a/packages/validation/src/command-adapter.ts b/packages/validation/src/command-adapter.ts index 8e615e7..9bd7e0c 100644 --- a/packages/validation/src/command-adapter.ts +++ b/packages/validation/src/command-adapter.ts @@ -56,17 +56,19 @@ export function createCheckCommandAdapter(options: ValidationCommandAdapterOptio return async (request) => { try { const parsed = parseCheckCommandOptions(request.args); + const repoRoot = repoRootForCommand(parsed, options); + const availableChecks = registryChecks(options, repoRoot); if (parsed.route === "manifest") { - return manifestCommandResult(request, registryChecks(options, repoRootForCommand(parsed, options)), "check manifest"); + return manifestCommandResult(request, availableChecks, "check manifest"); } const validationRequest = normalizeValidationRequest( { repo: { - repoRoot: repoRootForCommand(parsed, options) + repoRoot }, scope: requireScope(parsed), graph: { - mode: parsed.graphMode, + mode: forcedGraphMode(parsed.checks, parsed.graphMode, availableChecks), provider: defaultValidationGraphProvider }, overlays: [], @@ -92,14 +94,15 @@ export function createValidateCommandAdapter(options: ValidationCommandAdapterOp return async (request) => { try { const parsed = parseValidateCommandOptions(request.args); + const repoRoot = repoRootForCommand(parsed, options); if (parsed.route === "manifest") { - return manifestCommandResult(request, registryChecks(options, repoRootForCommand(parsed, options)), "validate manifest"); + return manifestCommandResult(request, registryChecks(options, repoRoot), "validate manifest"); } if (parsed.route === "pre-write") return preWriteValidationCommandResult(request, parsed, options); const payload = await readRequestPayload(parsed.requestFile); const validatedPayload = validateValidationRequestPayload(payload); const validationRequest = normalizeValidationRequest( - applyValidateCommandOverrides(validatedPayload, parsed), + applyValidateCommandOverrides(validatedPayload, parsed, options), { provider: defaultValidationGraphProvider } @@ -123,7 +126,7 @@ async function preWriteValidationCommandResult( try { const payload = await readRequestPayload(parsed.requestFile); const validatedPayload = validateValidationRequestPayload(payload); - validationRequest = normalizeValidationRequest(applyValidateCommandOverrides(validatedPayload, parsed), { + validationRequest = normalizeValidationRequest(applyValidateCommandOverrides(validatedPayload, parsed, options), { provider: defaultValidationGraphProvider }); result = await runRequestWithTimeout(validationRequest, parsed, options, timeoutMs); @@ -394,7 +397,8 @@ function registryChecks(options: ValidationCommandAdapterOptions, repoRoot: stri function applyValidateCommandOverrides( request: ValidationRequest, - parsed: ParsedValidationCommandOptions + parsed: ParsedValidationCommandOptions, + options: ValidationCommandAdapterOptions ): ValidationRequest { let next = request; if (parsed.repoRoot !== undefined) { @@ -418,9 +422,15 @@ function applyValidateCommandOverrides( }; } if (parsed.checks !== undefined) { + const repoRoot = repoRootForRequest(next, parsed, options); + const availableChecks = registryChecks(options, repoRoot); next = { ...next, - checks: parsed.checks + checks: parsed.checks, + graph: { + ...next.graph, + mode: forcedGraphMode(parsed.checks, next.graph.mode, availableChecks) + } }; } if (parsed.reportMode !== undefined) { @@ -432,6 +442,16 @@ function applyValidateCommandOverrides( return next; } +function forcedGraphMode( + checks: readonly string[] | undefined, + current: ValidationRequest["graph"]["mode"], + availableChecks: readonly ValidationCheckDefinition[] +): ValidationRequest["graph"]["mode"] { + if (checks === undefined) return current; + const byId = new Map(availableChecks.map((check) => [check.id, check])); + return checks.some((checkId) => byId.get(checkId)?.requiresGraph === true) ? "required" : current; +} + function repoIdentityForOverride(repo: ValidationRequest["repo"], repoRoot: string): ValidationRequest["repo"] { const { repoId: _repoId, ...rest } = repo; return { diff --git a/packages/validation/src/runner.ts b/packages/validation/src/runner.ts index 493c832..f24042f 100644 --- a/packages/validation/src/runner.ts +++ b/packages/validation/src/runner.ts @@ -8,8 +8,8 @@ import type { ValidationFailureCategory, ValidationRequest, ValidationResult, - PythonProjectContext, PythonValidationCapabilityRun, + PythonProjectContext, ValidationScopeKind, ValidationSkippedCheck } from "@the-open-engine/opcore-contracts"; @@ -559,6 +559,7 @@ async function executeSelectedChecks(args: ExecuteChecksArgs): Promise 0, true, label); + if (label === "workspace-isolation") { + assert.equal(capabilityRun?.collection?.stage, "collection"); + assert.equal(capabilityRun?.execution?.stage, "execution"); + } + break; + case "fail": + assert.equal(validationResult?.status, "policy_failure", label); + assert.equal(capabilityRun?.outcome, "findings", label); + assert.equal((capabilityRun?.counts?.failedCount ?? 0) + (capabilityRun?.counts?.errorCount ?? 0) > 0, true, label); + break; + case "collection-fail": + assert.equal(validationResult?.status, "policy_failure", label); + assert.equal(capabilityRun?.collection?.termination, "exited", label); + assert.equal(capabilityRun?.outcome !== "passed", true, label); + assert.equal(Object.hasOwn(capabilityRun, "execution"), false, label); + break; + case "timeout": + assert.equal(validationResult?.status, "policy_failure", label); + assert.equal(capabilityRun?.outcome, "timeout", label); + assert.equal(capabilityRun?.execution?.termination, "timeout", label); + break; + case "manifest": + assert.equal(validationResult?.status, "passed", label); + assert.equal(capabilityRun?.outcome, "passed", label); + assert.equal(capabilityRun?.selectionMode, "manifest", label); + assert.equal((capabilityRun?.counts?.passedCount ?? 0) > 100, true, label); + break; + default: + throw new Error(`Unknown scenario: ${label}`); + } +} + +function summarize(label, validationResult, capabilityRun) { + return { + name: label, + status: validationResult.status, + outcome: capabilityRun.outcome, + activation: capabilityRun.activation, + selectionMode: capabilityRun.selectionMode, + counts: capabilityRun.counts, + cleanup: capabilityRun.cleanup, + collection: capabilityRun.collection, + execution: capabilityRun.execution + }; +} + +function installAmbientPlugin(venvPython) { + const sitePackages = run(venvPython, ["-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], { cwd: repoRoot }).stdout.trim(); + const distInfo = join(sitePackages, "opcore_bad_pytest_plugin-0.0.0.dist-info"); + mkdirSync(distInfo, { recursive: true }); + writeFileSync(join(sitePackages, "opcore_bad_pytest_plugin.py"), "raise RuntimeError('ambient pytest plugin loaded')\n"); + writeFileSync(join(distInfo, "METADATA"), [ + "Metadata-Version: 2.1", + "Name: opcore-bad-pytest-plugin", + "Version: 0.0.0", + "" + ].join("\n")); + writeFileSync(join(distInfo, "entry_points.txt"), [ + "[pytest11]", + "opcore_bad_plugin = opcore_bad_pytest_plugin", + "" + ].join("\n")); +} + +function materializeFixture(name, destinationName = name) { + const source = join(fixtureRoot, name); + const repo = join(tempRoot, destinationName); + cpSync(source, repo, { recursive: true }); + for (const path of walkFiles(repo)) { + if (!path.endsWith(".fixture")) continue; + renameSync(path, path.slice(0, -".fixture".length)); + } + const localBin = join(repo, ".venv", "bin"); + mkdirSync(localBin, { recursive: true }); + for (const tool of ["python", "python3", "pytest"]) { + const wrapperPath = join(localBin, tool); + writeFileSync(wrapperPath, `#!/bin/sh\nexec "${join(tempRoot, "venv", "bin", tool)}" "$@"\n`); + chmodSync(wrapperPath, 0o755); + } + return repo; +} + +function currentPytestTempRoots() { + return new Set( + readdirSync(tmpdir(), { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && entry.name.startsWith("opcore-python-pytest-")) + .map((entry) => join(tmpdir(), entry.name)) + ); +} + +function assertNoNewPytestTempRoots(before, label) { + const after = currentPytestTempRoots(); + const leaked = [...after].filter((path) => !before.has(path)); + assert.deepEqual(leaked, [], `${label} leaked pytest temp roots`); +} + +function assertNoPytestTempLeaks(before) { + const after = currentPytestTempRoots(); + const leaked = [...after].filter((path) => !before.has(path)); + assert.deepEqual(leaked, [], "pytest authority proof leaked temp roots"); +} + +function repoDigest(root) { + const hash = createHash("sha256"); + for (const path of walkFiles(root)) { + hash.update(path.slice(root.length + 1)); + hash.update("\0"); + hash.update(readFileSync(path)); + hash.update("\0"); + } + return `sha256:${hash.digest("hex")}`; +} + +function walkFiles(root) { + const entries = []; + for (const dirent of readdirSync(root, { withFileTypes: true })) { + if ([".git", ".opcore", ".pytest_cache", "__pycache__"].includes(dirent.name)) continue; + const path = join(root, dirent.name); + if (dirent.isDirectory()) entries.push(...walkFiles(path)); + else if (dirent.isFile()) entries.push(path); + } + return entries.sort(); +} + +function run(command, args, options) { + const result = spawnSync(command, args, { + cwd: options.cwd, + env: options.env, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"] + }); + const allowedExitCodes = options.allowedExitCodes ?? [0]; + if (!allowedExitCodes.includes(result.status ?? -1)) { + throw new Error(`${command} ${args.join(" ")} failed\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`); + } + return result; +} diff --git a/scripts/generate-cutover-receipt.mjs b/scripts/generate-cutover-receipt.mjs index 30b9ddf..93f7b4d 100644 --- a/scripts/generate-cutover-receipt.mjs +++ b/scripts/generate-cutover-receipt.mjs @@ -517,7 +517,7 @@ function runPythonToolDegradationNegativeChecks(tempRoot, opcoreBin, env, comman throw new Error("python-relevant-tests-no-pytest did not pass with graph-backed relevant-test evidence"); } const relevantTestCodes = relevantTestsParsed.validationResult.diagnostics?.map((diagnostic) => diagnostic.code) ?? []; - if (!relevantTestCodes.some((code) => code === "PY_RELEVANT_TESTS_FOUND" || code === "PY_RELEVANT_TESTS_ABSENT")) { + if (!relevantTestCodes.some((code) => code === "PY_RELEVANT_TEST_CANDIDATES_FOUND" || code === "PY_RELEVANT_TEST_CANDIDATES_ABSENT")) { throw new Error("python-relevant-tests-no-pytest did not report Python relevant-test graph evidence"); } const statusResult = run(opcoreBin, ["status", "--json"], { diff --git a/tests/asp-provider.test.mjs b/tests/asp-provider.test.mjs index 4607fa7..9c875d6 100644 --- a/tests/asp-provider.test.mjs +++ b/tests/asp-provider.test.mjs @@ -39,6 +39,7 @@ const allCheckIds = [ "python.import-graph", "python.dead-code", "python.relevant-tests", + "python.pytest", "docs.existence", "docs.staleness", "docs.freshness", diff --git a/tests/command-router.test.mjs b/tests/command-router.test.mjs index c2d069d..137ee23 100644 --- a/tests/command-router.test.mjs +++ b/tests/command-router.test.mjs @@ -38,7 +38,8 @@ const requiredDefaultCheckIds = [ "python.types", "python.import-graph", "python.dead-code", - "python.relevant-tests" + "python.relevant-tests", + "python.pytest" ]; describe("Opcore command router", () => { diff --git a/tests/conformance.test.mjs b/tests/conformance.test.mjs index 547a67d..3c8d1c7 100644 --- a/tests/conformance.test.mjs +++ b/tests/conformance.test.mjs @@ -405,7 +405,8 @@ describe("conformance fixture metadata", () => { "python.types", "python.import-graph", "python.dead-code", - "python.relevant-tests" + "python.relevant-tests", + "python.pytest" ]); assert.deepEqual(validation.degradedTools, ["mypy", "pyright", "ruff", "pytest"]); }); diff --git a/tests/fixtures/package-packlists.json b/tests/fixtures/package-packlists.json index 7dffd92..df6cd14 100644 --- a/tests/fixtures/package-packlists.json +++ b/tests/fixtures/package-packlists.json @@ -409,6 +409,9 @@ "node_modules/@the-open-engine/opcore-validation-python/dist/ini-config.d.ts", "node_modules/@the-open-engine/opcore-validation-python/dist/ini-config.d.ts.map", "node_modules/@the-open-engine/opcore-validation-python/dist/ini-config.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/materialized-workspace.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/materialized-workspace.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/materialized-workspace.js", "node_modules/@the-open-engine/opcore-validation-python/dist/mypy-config-values.d.ts", "node_modules/@the-open-engine/opcore-validation-python/dist/mypy-config-values.d.ts.map", "node_modules/@the-open-engine/opcore-validation-python/dist/mypy-config-values.js", @@ -472,6 +475,30 @@ "node_modules/@the-open-engine/opcore-validation-python/dist/pyright-runner.d.ts", "node_modules/@the-open-engine/opcore-validation-python/dist/pyright-runner.d.ts.map", "node_modules/@the-open-engine/opcore-validation-python/dist/pyright-runner.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-check.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-check.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-check.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-hook-source.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-hook-source.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-hook-source.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-process.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-process.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-process.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-project-runner.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-project-runner.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-project-runner.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-protocol.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-protocol.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-protocol.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-result.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-result.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-result.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-types.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-types.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-types.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-workspace.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-workspace.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-workspace.js", "node_modules/@the-open-engine/opcore-validation-python/dist/relevant-tests-check.d.ts", "node_modules/@the-open-engine/opcore-validation-python/dist/relevant-tests-check.d.ts.map", "node_modules/@the-open-engine/opcore-validation-python/dist/relevant-tests-check.js", diff --git a/tests/installed-bins.test.mjs b/tests/installed-bins.test.mjs index 27ce8fe..fbb11bc 100644 --- a/tests/installed-bins.test.mjs +++ b/tests/installed-bins.test.mjs @@ -170,6 +170,7 @@ describe("installed package bins", () => { "python.import-graph", "python.dead-code", "python.relevant-tests", + "python.pytest", "docs.existence", "docs.staleness", "docs.freshness", diff --git a/tests/opcore-metrics.test.mjs b/tests/opcore-metrics.test.mjs index e0f0750..579b74f 100644 --- a/tests/opcore-metrics.test.mjs +++ b/tests/opcore-metrics.test.mjs @@ -507,8 +507,8 @@ function validationResult() { category: "test", severity: "info", path: "pkg/untested.py", - code: "PY_RELEVANT_TESTS_ABSENT", - message: "No TESTED_BY graph evidence found." + code: "PY_RELEVANT_TEST_CANDIDATES_ABSENT", + message: "No TESTED_BY graph candidate evidence found." }, { category: "graph", @@ -577,7 +577,8 @@ function validationResult() { "python.types", "python.import-graph", "python.dead-code", - "python.relevant-tests" + "python.relevant-tests", + "python.pytest" ], runs: [ { diff --git a/tests/validation-cli.test.mjs b/tests/validation-cli.test.mjs index 10a157e..dd72d01 100644 --- a/tests/validation-cli.test.mjs +++ b/tests/validation-cli.test.mjs @@ -41,7 +41,8 @@ const pythonCheckIds = [ "python.types", "python.import-graph", "python.dead-code", - "python.relevant-tests" + "python.relevant-tests", + "python.pytest" ]; const optInPythonCheckIds = ["python.ruff-lint", "python.ruff-format"]; const docsCheckIds = [ @@ -58,7 +59,8 @@ const docsCheckIds = [ ]; const cloneCheckIds = ["clone.duplication"]; const typeScriptExecutableDefaultCheckIds = typeScriptCheckIds.filter((checkId) => checkId !== "typescript.lint"); -const executableDefaultCheckIds = [...typeScriptExecutableDefaultCheckIds, ...rustCheckIds, ...pythonCheckIds, ...cloneCheckIds]; +const pythonExecutableDefaultCheckIds = pythonCheckIds.filter((checkId) => checkId !== "python.pytest"); +const executableDefaultCheckIds = [...typeScriptExecutableDefaultCheckIds, ...rustCheckIds, ...pythonExecutableDefaultCheckIds, ...cloneCheckIds]; const defaultCheckIds = [...typeScriptCheckIds, ...rustCheckIds, ...pythonCheckIds, ...docsCheckIds, ...cloneCheckIds]; const availableCheckIds = [ ...typeScriptCheckIds, @@ -868,6 +870,40 @@ describe("validation CLI", () => { } }); + it("returns a typed disabled python.pytest capability run instead of an unknown check failure", () => { + const temp = mkdtempSync(join(tmpdir(), "opcore-python-pytest-disabled-")); + try { + mkdirSync(join(temp, "src"), { recursive: true }); + writeFileSync(join(temp, "src/app.py"), "VALUE = 1\n"); + writeRepoConfigObject(temp, { + schemaVersion: 1, + kind: "opcore_init_config", + validation: { + checks: { + disabled: ["python.pytest"] + } + } + }); + + const result = JSON.parse(runRaw(["check", "files", "--files", "src/app.py", "--repo", temp, "--checks", "python.pytest", "--json"], [0]).stdout); + + assert.equal(result.validationResult.status, "passed"); + assert.deepEqual(result.validationResult.manifest.checks, ["python.pytest"]); + assert.deepEqual(result.validationResult.pythonCapabilityRuns, [ + { + capability: "pytest", + checkId: "python.pytest", + activation: "disabled", + outcome: "disabled", + message: "python.pytest is disabled by repo policy.", + selectionMode: "none" + } + ]); + } finally { + rmSync(temp, { recursive: true, force: true }); + } + }); + it("uses native docs enabled flags for default docs checks", () => { const temp = mkdtempSync(join(tmpdir(), "opcore-native-default-docs-checks-")); try { diff --git a/tests/validation-python.test.mjs b/tests/validation-python.test.mjs index 47235cb..808d07b 100644 --- a/tests/validation-python.test.mjs +++ b/tests/validation-python.test.mjs @@ -10,6 +10,7 @@ import { validationChecksForRepoPolicy } from "../packages/validation-policy/dis import { PYTHON_DEAD_CODE_CHECK_ID, PYTHON_IMPORT_GRAPH_CHECK_ID, + PYTHON_PYTEST_CHECK_ID, PYTHON_RELEVANT_TESTS_CHECK_ID, PYTHON_RUFF_FORMAT_CHECK_ID, PYTHON_RUFF_LINT_CHECK_ID, @@ -78,12 +79,14 @@ describe("validation-python adapter", () => { PYTHON_TYPES_CHECK_ID, PYTHON_IMPORT_GRAPH_CHECK_ID, PYTHON_DEAD_CODE_CHECK_ID, - PYTHON_RELEVANT_TESTS_CHECK_ID + PYTHON_RELEVANT_TESTS_CHECK_ID, + PYTHON_PYTEST_CHECK_ID ] ); assert.equal(registry.byId.get(PYTHON_SYNTAX_CHECK_ID)?.requiresGraph, false); assert.equal(registry.byId.get(PYTHON_SOURCE_HYGIENE_CHECK_ID)?.requiresGraph, false); assert.equal(registry.byId.get(PYTHON_IMPORT_GRAPH_CHECK_ID)?.requiresGraph, true); + assert.deepEqual(registry.byId.get(PYTHON_PYTEST_CHECK_ID)?.defaultScopes, []); }); it("places canonical explicit Ruff config overrides after the subcommand", () => { @@ -4519,7 +4522,7 @@ describe("validation-python adapter", () => { assert.equal(result.status, "passed", JSON.stringify(result, null, 2)); assert.deepEqual(result.diagnostics.map((diagnostic) => diagnostic.code).sort(), [ "PY_DEAD_CODE_UNSUPPORTED", - "PY_RELEVANT_TESTS_ABSENT" + "PY_RELEVANT_TEST_CANDIDATES_ABSENT" ]); }); @@ -4759,7 +4762,121 @@ describe("validation-python adapter", () => { ); assert.equal(result.status, "passed"); - assert.deepEqual(result.diagnostics.map((diagnostic) => diagnostic.code), ["PY_RELEVANT_TESTS_FOUND"]); + assert.deepEqual(result.diagnostics.map((diagnostic) => diagnostic.code), ["PY_RELEVANT_TEST_CANDIDATES_FOUND"]); + }); + + it("materializes pytest support files and preserves exact regular-file modes", async () => { + const repoRoot = mkdtempSync(join(tmpdir(), "opcore-python-pytest-support-")); + try { + writePassingPythonProtocolShim(repoRoot); + writePytestShim(repoRoot, { + collectionShellLines: [ + "emit '{\"type\":\"collected\",\"nodeid\":\"tests/test_app.py::test_support_files\"}'", + "emit '{\"type\":\"session_finish\",\"exitstatus\":0}'" + ], + executionShellLines: [ + "node -e \"const fs=require('fs'); const payload=JSON.parse(fs.readFileSync('tests/data/payload.json','utf8')); if(payload.value!==7) { console.error('support file missing'); process.exit(9); } if((fs.statSync('tests/private.txt').mode & 0o777)!==0o640) { console.error('private mode mismatch'); process.exit(10); } if((fs.statSync('tests/run-me.sh').mode & 0o777)!==0o750) { console.error('exec mode mismatch'); process.exit(11); }\"", + "emit '{\"type\":\"collected\",\"nodeid\":\"tests/test_app.py::test_support_files\"}'", + "emit '{\"type\":\"test_report\",\"nodeid\":\"tests/test_app.py::test_support_files\",\"when\":\"call\",\"outcome\":\"passed\"}'", + "emit '{\"type\":\"session_finish\",\"exitstatus\":0}'" + ] + }); + const files = { + "pyproject.toml": "[project]\nname='fixture'\n", + "src/app.py": "VALUE = 7\n", + "tests/test_app.py": "def test_support_files():\n assert True\n", + "tests/data/payload.json": "{\"value\":7}\n", + "tests/private.txt": "secret\n", + "tests/run-me.sh": "#!/bin/sh\nexit 0\n" + }; + writeRepoFiles(repoRoot, files); + chmodSync(join(repoRoot, "tests/private.txt"), 0o640); + chmodSync(join(repoRoot, "tests/run-me.sh"), 0o750); + const result = await runner({ + files, + checks: createPythonValidationChecks({ + repoRoot, + env: { PATH: "" }, + nodeWorkspace: createNodePythonProjectWorkspace(repoRoot), + processProbe: successfulProbe(), + toolArgv: { pytest: [join(repoRoot, ".venv", "bin", "pytest")] } + }), + graphProviderClient: graphClient({ + factQuery: (query) => + availableFactResult( + query, + [], + query.selector.kind === "edges" && query.selector.edgeKinds?.includes("TESTED_BY") + ? [{ kind: "TESTED_BY", from: "file:src/app.py", to: "file:tests/test_app.py" }] + : [] + ) + }) + }).runValidation(request({ + repo: { repoRoot }, + checks: [PYTHON_PYTEST_CHECK_ID], + scope: { kind: "files", files: ["src/app.py"] } + })); + + assert.equal(result.status, "passed", JSON.stringify(result, null, 2)); + assert.equal(result.pythonCapabilityRuns[0]?.outcome, "passed"); + assert.equal(result.pythonCapabilityRuns[0]?.counts?.passedCount, 1); + } finally { + rmSync(repoRoot, { recursive: true, force: true }); + } + }); + + it("fails python.pytest when execution hook events do not match the collected node ids", async () => { + const repoRoot = mkdtempSync(join(tmpdir(), "opcore-python-pytest-mismatch-")); + try { + writePassingPythonProtocolShim(repoRoot); + writePytestShim(repoRoot, { + collectionShellLines: [ + "emit '{\"type\":\"collected\",\"nodeid\":\"tests/test_app.py::test_expected\"}'", + "emit '{\"type\":\"session_finish\",\"exitstatus\":0}'" + ], + executionShellLines: [ + "emit '{\"type\":\"collected\",\"nodeid\":\"tests/test_app.py::test_expected\"}'", + "emit '{\"type\":\"test_report\",\"nodeid\":\"tests/test_app.py::test_other\",\"when\":\"call\",\"outcome\":\"passed\"}'", + "emit '{\"type\":\"session_finish\",\"exitstatus\":0}'" + ] + }); + const files = { + "pyproject.toml": "[project]\nname='fixture'\n", + "src/app.py": "VALUE = 1\n", + "tests/test_app.py": "def test_expected():\n assert True\n" + }; + writeRepoFiles(repoRoot, files); + const result = await runner({ + files, + checks: createPythonValidationChecks({ + repoRoot, + env: { PATH: "" }, + nodeWorkspace: createNodePythonProjectWorkspace(repoRoot), + processProbe: successfulProbe(), + toolArgv: { pytest: [join(repoRoot, ".venv", "bin", "pytest")] } + }), + graphProviderClient: graphClient({ + factQuery: (query) => + availableFactResult( + query, + [], + query.selector.kind === "edges" && query.selector.edgeKinds?.includes("TESTED_BY") + ? [{ kind: "TESTED_BY", from: "file:src/app.py", to: "file:tests/test_app.py" }] + : [] + ) + }) + }).runValidation(request({ + repo: { repoRoot }, + checks: [PYTHON_PYTEST_CHECK_ID], + scope: { kind: "files", files: ["src/app.py"] } + })); + + assert.equal(result.status, "policy_failure", JSON.stringify(result, null, 2)); + assert.equal(result.pythonCapabilityRuns[0]?.outcome, "tool_failure"); + assert.equal(result.diagnostics.every((diagnostic) => diagnostic.code === "PYTHON_PYTEST_PROTOCOL_MISMATCH"), true); + } finally { + rmSync(repoRoot, { recursive: true, force: true }); + } }); it("passes clean Python validation fixture checks", async () => { @@ -6251,6 +6368,38 @@ function rawPyrightShim(version, body) { ].join("\n"); } +function writePytestShim(repoRoot, options) { + writeToolShim( + repoRoot, + "pytest", + [ + "#!/bin/sh", + "set -eu", + "output=\"$OPCORE_PYTEST_HOOK_OUTPUT\"", + "collect=0", + "for arg in \"$@\"; do", + " if [ \"$arg\" = \"--version\" ]; then echo 'pytest 8.4.1'; exit 0; fi", + " if [ \"$arg\" = \"--collect-only\" ]; then collect=1; fi", + "done", + "emit() { printf '%s\\n' \"$1\" >> \"$output\"; }", + "if [ \"$collect\" = \"1\" ]; then", + ...options.collectionShellLines.map((line) => ` ${line}`), + ` exit ${options.collectionExitCode ?? 0}`, + "fi", + ...options.executionShellLines.map((line) => line), + `exit ${options.executionExitCode ?? 0}`, + "" + ].join("\n") + ); +} + +function writeRepoFiles(repoRoot, files) { + for (const [path, content] of Object.entries(files)) { + mkdirSync(dirname(join(repoRoot, path)), { recursive: true }); + writeFileSync(join(repoRoot, path), content); + } +} + function writePythonProtocolShim(repoRoot, compilerBranch) { const bin = join(repoRoot, ".venv", "bin"); mkdirSync(bin, { recursive: true });